From bb35e76f6056131cac9961f131c6578339832548 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 17 Oct 2023 09:58:15 -0400 Subject: [PATCH 1/5] fix(compiler): return error on bad regular expressions that fail to compile in Purge (#985) * init commit with go mod local replace * pull in types commit * add tests for auto-cancel types change --- compiler/native/compile_test.go | 22 +++++++++++++++++++++ compiler/native/expand.go | 5 ++++- compiler/native/parse_test.go | 4 ++++ compiler/native/testdata/steps_pipeline.yml | 2 ++ compiler/native/transform.go | 14 +++++++++++-- compiler/native/transform_test.go | 12 +++++++---- go.mod | 2 +- go.sum | 4 ++-- 8 files changed, 55 insertions(+), 10 deletions(-) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index ed5627027..91948e07b 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -110,6 +110,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Stages: pipeline.StageSlice{ &pipeline.Stage{ @@ -482,6 +483,10 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{ + Running: true, + Pending: true, + }, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -693,6 +698,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Stages: pipeline.StageSlice{ &pipeline.Stage{ @@ -960,6 +966,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1155,6 +1162,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1275,6 +1283,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1449,6 +1458,7 @@ func TestNative_Compile_Clone(t *testing.T) { Clone: false, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1479,6 +1489,7 @@ func TestNative_Compile_Clone(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1518,6 +1529,7 @@ func TestNative_Compile_Clone(t *testing.T) { Clone: false, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1638,6 +1650,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1683,6 +1696,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -1728,6 +1742,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { Clone: true, Template: false, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: pipeline.ContainerSlice{ &pipeline.Container{ @@ -2232,6 +2247,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Stages: []*pipeline.Stage{ { @@ -2387,6 +2403,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Stages: []*pipeline.Stage{ { @@ -2560,6 +2577,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: []*pipeline.Container{ { @@ -2676,6 +2694,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: []*pipeline.Container{ { @@ -2773,6 +2792,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: []*pipeline.Container{ { @@ -2848,6 +2868,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Steps: []*pipeline.Container{ { @@ -2892,6 +2913,7 @@ func Test_Compile_Inline(t *testing.T) { Metadata: pipeline.Metadata{ Clone: true, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{}, }, Stages: []*pipeline.Stage{ { diff --git a/compiler/native/expand.go b/compiler/native/expand.go index e02b0f0c0..c0173417b 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -89,7 +89,10 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * Steps: *check.ToPipeline(), } - pipeline = pipeline.Purge(r) + pipeline, err := pipeline.Purge(r) + if err != nil { + return nil, fmt.Errorf("unable to purge pipeline: %w", err) + } // if step purged, do not proceed with expansion if len(pipeline.Steps) == 0 { diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index 9698a103d..672ebcaaf 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -348,6 +348,7 @@ func TestNative_Parse_StagesPipeline(t *testing.T) { func TestNative_Parse_StepsPipeline(t *testing.T) { // setup types + tBool := true client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", @@ -355,6 +356,9 @@ func TestNative_Parse_StepsPipeline(t *testing.T) { Template: false, Clone: nil, Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &yaml.CancelOptions{ + Running: &tBool, + }, }, Environment: map[string]string{ "HELLO": "Hello, Global Environment", diff --git a/compiler/native/testdata/steps_pipeline.yml b/compiler/native/testdata/steps_pipeline.yml index aa28e2e21..5f6e22ffd 100644 --- a/compiler/native/testdata/steps_pipeline.yml +++ b/compiler/native/testdata/steps_pipeline.yml @@ -3,6 +3,8 @@ version: "1" metadata: template: false + auto_cancel: + running: true environment: HELLO: "Hello, Global Environment" diff --git a/compiler/native/transform.go b/compiler/native/transform.go index 93844bee0..1165e6a40 100644 --- a/compiler/native/transform.go +++ b/compiler/native/transform.go @@ -113,7 +113,12 @@ func (c *client) TransformStages(r *pipeline.RuleData, p *yaml.Build) (*pipeline secret.Origin.ID = pattern } - return pipeline.Purge(r), nil + build, err := pipeline.Purge(r) + if err != nil { + return nil, fmt.Errorf("unable to purge pipeline: %w", err) + } + + return build, nil } // TransformSteps converts a yaml configuration with steps into an executable pipeline. @@ -192,5 +197,10 @@ func (c *client) TransformSteps(r *pipeline.RuleData, p *yaml.Build) (*pipeline. secret.Origin.ID = pattern } - return pipeline.Purge(r), nil + build, err := pipeline.Purge(r) + if err != nil { + return nil, fmt.Errorf("unable to purge pipeline: %w", err) + } + + return build, nil } diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index 1b50905ed..743f9a46c 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -112,7 +112,8 @@ func TestNative_TransformStages(t *testing.T) { ID: "__0", Version: "v1", Metadata: pipeline.Metadata{ - Clone: true, + Clone: true, + AutoCancel: &pipeline.CancelOptions{}, }, Services: pipeline.ContainerSlice{ &pipeline.Container{ @@ -167,7 +168,8 @@ func TestNative_TransformStages(t *testing.T) { ID: "localOrg_localRepo_1", Version: "v1", Metadata: pipeline.Metadata{ - Clone: true, + Clone: true, + AutoCancel: &pipeline.CancelOptions{}, }, Services: pipeline.ContainerSlice{ &pipeline.Container{ @@ -339,7 +341,8 @@ func TestNative_TransformSteps(t *testing.T) { ID: "__0", Version: "v1", Metadata: pipeline.Metadata{ - Clone: true, + Clone: true, + AutoCancel: &pipeline.CancelOptions{}, }, Services: pipeline.ContainerSlice{ &pipeline.Container{ @@ -389,7 +392,8 @@ func TestNative_TransformSteps(t *testing.T) { ID: "localOrg_localRepo_1", Version: "v1", Metadata: pipeline.Metadata{ - Clone: true, + Clone: true, + AutoCancel: &pipeline.CancelOptions{}, }, Services: pipeline.ContainerSlice{ &pipeline.Container{ diff --git a/go.mod b/go.mod index 939642d85..1aa96d380 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.21.0 + github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v55 v55.0.0 diff --git a/go.sum b/go.sum index 8f6fd796a..ccc23bbaa 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-vela/types v0.21.0 h1:yZrVUw4jKO0JHaUBkOIZZdniDGyDOpTMbKriemdm1jg= -github.com/go-vela/types v0.21.0/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ= +github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af h1:qiP6pXFDyPDDP+hy8zY+nhmoWv9aoQrrnNmfAAT6yCA= +github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= From d90fdb7c11ad8613cd7be3939bbc38e9374264b1 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 17 Oct 2023 11:34:10 -0400 Subject: [PATCH 2/5] fix(admin/clean): clean executable when build is cleaned (#988) * fix(admin/clean): clean executable when build is cleaned * report number of rows affected * integration test with rows affected --------- Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- api/admin/clean.go | 16 ++- database/executable/clean.go | 41 ++++++++ database/executable/clean_test.go | 162 ++++++++++++++++++++++++++++++ database/executable/interface.go | 2 + database/integration_test.go | 28 ++++++ 5 files changed, 246 insertions(+), 3 deletions(-) create mode 100644 database/executable/clean.go create mode 100644 database/executable/clean_test.go diff --git a/api/admin/clean.go b/api/admin/clean.go index 39d8c32a4..efe3819c6 100644 --- a/api/admin/clean.go +++ b/api/admin/clean.go @@ -108,10 +108,20 @@ func CleanResources(c *gin.Context) { logrus.Infof("platform admin %s: cleaned %d builds in database", u.GetName(), builds) + // clean executables + executables, err := database.FromContext(c).CleanBuildExecutables(ctx) + if err != nil { + retErr := fmt.Errorf("%d builds cleaned. unable to clean build executables: %w", builds, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + // clean services services, err := database.FromContext(c).CleanServices(ctx, msg, before) if err != nil { - retErr := fmt.Errorf("%d builds cleaned. unable to update services: %w", builds, err) + retErr := fmt.Errorf("%d builds cleaned. %d executables cleaned. unable to update services: %w", builds, executables, err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -123,7 +133,7 @@ func CleanResources(c *gin.Context) { // clean steps steps, err := database.FromContext(c).CleanSteps(msg, before) if err != nil { - retErr := fmt.Errorf("%d builds cleaned. %d services cleaned. unable to update steps: %w", builds, services, err) + retErr := fmt.Errorf("%d builds cleaned. %d executables cleaned. %d services cleaned. unable to update steps: %w", builds, executables, services, err) util.HandleError(c, http.StatusInternalServerError, retErr) @@ -132,5 +142,5 @@ func CleanResources(c *gin.Context) { logrus.Infof("platform admin %s: cleaned %d steps in database", u.GetName(), steps) - c.JSON(http.StatusOK, fmt.Sprintf("%d builds cleaned. %d services cleaned. %d steps cleaned.", builds, services, steps)) + c.JSON(http.StatusOK, fmt.Sprintf("%d builds cleaned. %d executables cleaned. %d services cleaned. %d steps cleaned.", builds, executables, services, steps)) } diff --git a/database/executable/clean.go b/database/executable/clean.go new file mode 100644 index 000000000..34579f778 --- /dev/null +++ b/database/executable/clean.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executable + +import ( + "context" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" +) + +const CleanExecutablesPostgres = ` + DELETE FROM build_executables + USING builds + WHERE builds.id = build_executables.build_id + AND builds.status = 'error'; +` + +const CleanExecutablesSqlite = ` + DELETE FROM build_executables + WHERE build_id IN ( + SELECT build_id FROM build_executables e + INNER JOIN builds b + ON e.build_id=b.id + WHERE b.status = 'error' + ); +` + +// CleanBuildExecutables pops executables which have a corresponding build that was cleaned. +func (e *engine) CleanBuildExecutables(ctx context.Context) (int64, error) { + logrus.Trace("clearing build executables in the database") + + switch e.config.Driver { + case constants.DriverPostgres: + res := e.client.Exec(CleanExecutablesPostgres) + return res.RowsAffected, res.Error + default: + res := e.client.Exec(CleanExecutablesSqlite) + return res.RowsAffected, res.Error + } +} diff --git a/database/executable/clean_test.go b/database/executable/clean_test.go new file mode 100644 index 000000000..edcb0cf21 --- /dev/null +++ b/database/executable/clean_test.go @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executable + +import ( + "context" + "fmt" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +func TestExecutable_Engine_CleanExecutables(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetStatus("pending") + _buildOne.SetCreated(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetStatus("error") + _buildTwo.SetCreated(1) + _buildTwo.SetDeployPayload(nil) + + _bExecutableOne := testBuildExecutable() + _bExecutableOne.SetID(1) + _bExecutableOne.SetBuildID(1) + _bExecutableOne.SetData([]byte("foo")) + + _bExecutableTwo := testBuildExecutable() + _bExecutableTwo.SetID(2) + _bExecutableTwo.SetBuildID(2) + _bExecutableTwo.SetData([]byte("bar")) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec("DELETE FROM build_executables USING builds WHERE builds.id = build_executables.build_id AND builds.status = 'error';"). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _mock.ExpectQuery(`DELETE FROM "build_executables" WHERE build_id = $1 RETURNING *`).WithArgs(2).WillReturnError(fmt.Errorf("not found")) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateBuildExecutable(context.TODO(), _bExecutableOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.CreateBuildExecutable(context.TODO(), _bExecutableTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&database.Build{}) + if err != nil { + t.Errorf("unable to create repo table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableBuild).Create(database.BuildFromLibrary(_buildOne)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableBuild).Create(database.BuildFromLibrary(_buildTwo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CleanBuildExecutables(context.TODO()) + + if test.failure { + if err == nil { + t.Errorf("CleanExecutables for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CleanExecutables for %s returned err: %v", test.name, err) + } + + if got != 1 { + t.Errorf("CleanExecutables for %s should have affected 1 row, affected %d", test.name, got) + } + + _, err = test.database.PopBuildExecutable(context.TODO(), 2) + if err == nil { + t.Errorf("CleanExecutables for %s should have returned an error", test.name) + } + }) + } +} + +// testBuild is a test helper function to create a library +// Build type with all fields set to their zero values. +func testBuild() *library.Build { + return &library.Build{ + ID: new(int64), + RepoID: new(int64), + PipelineID: new(int64), + Number: new(int), + Parent: new(int), + Event: new(string), + EventAction: new(string), + Status: new(string), + Error: new(string), + Enqueued: new(int64), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Deploy: new(string), + Clone: new(string), + Source: new(string), + Title: new(string), + Message: new(string), + Commit: new(string), + Sender: new(string), + Author: new(string), + Email: new(string), + Link: new(string), + Branch: new(string), + Ref: new(string), + BaseRef: new(string), + HeadRef: new(string), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} diff --git a/database/executable/interface.go b/database/executable/interface.go index fbff5048a..ee0cd056f 100644 --- a/database/executable/interface.go +++ b/database/executable/interface.go @@ -20,6 +20,8 @@ type BuildExecutableInterface interface { // // https://en.wikipedia.org/wiki/Data_manipulation_language + // CleanBuildExecutables defines a function that deletes errored builds' corresponding executables. + CleanBuildExecutables(context.Context) (int64, error) // CreateBuildExecutable defines a function that creates a build executable. CreateBuildExecutable(context.Context, *library.BuildExecutable) error // PopBuildExecutable defines a function that gets and deletes a build executable. diff --git a/database/integration_test.go b/database/integration_test.go index 6aa55502d..bcf2fcd7f 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -423,6 +423,34 @@ func testExecutables(t *testing.T, db Interface, resources *Resources) { } methods["PopBuildExecutable"] = true + resources.Builds[0].SetStatus(constants.StatusError) + + _, err := db.UpdateBuild(context.TODO(), resources.Builds[0]) + if err != nil { + t.Errorf("unable to update build for clean executables test") + } + + err = db.CreateBuildExecutable(context.TODO(), resources.Executables[0]) + if err != nil { + t.Errorf("unable to create executable %d: %v", resources.Executables[0].GetID(), err) + } + + count, err := db.CleanBuildExecutables(context.TODO()) + if err != nil { + t.Errorf("unable to clean executable %d: %v", resources.Executables[0].GetID(), err) + } + + if count != 1 { + t.Errorf("CleanBuildExecutables should have affected 1 row, affected %d", count) + } + + _, err = db.PopBuildExecutable(context.TODO(), resources.Builds[0].GetID()) + if err == nil { + t.Errorf("build executable not cleaned") + } + + methods["CleanBuildExecutables"] = true + // ensure we called all the methods we expected to for method, called := range methods { if !called { From b51d5a2cc8c44508a97350418b0df5913acda539 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 20 Oct 2023 10:57:01 -0400 Subject: [PATCH 3/5] feat(auto-cancel): server-side logic for auto canceling obsolete builds (#911) * init commit * move auto cancel to build pkg * db test file for new func * integration test * linter and fmt debug statements * linter overlord * address feedback * publish before auto cancel and continue upon failure to cancel --- api/auth/validate_oauth.go | 4 +- api/build/auto_cancel.go | 181 ++++++++++++++++++ api/build/cancel.go | 14 +- api/queue/queue.go | 4 +- api/webhook/post.go | 34 ++++ database/build/interface.go | 2 + database/build/list_pending_running_repo.go | 45 +++++ .../build/list_pending_running_repo_test.go | 148 ++++++++++++++ database/integration_test.go | 13 ++ database/resource.go | 4 +- database/validate.go | 4 +- database/validate_test.go | 4 +- router/middleware/signing_test.go | 4 +- router/queue.go | 4 +- 14 files changed, 442 insertions(+), 23 deletions(-) create mode 100644 api/build/auto_cancel.go create mode 100644 database/build/list_pending_running_repo.go create mode 100644 database/build/list_pending_running_repo_test.go diff --git a/api/auth/validate_oauth.go b/api/auth/validate_oauth.go index a941050ee..5a31c0069 100644 --- a/api/auth/validate_oauth.go +++ b/api/auth/validate_oauth.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package auth diff --git a/api/build/auto_cancel.go b/api/build/auto_cancel.go new file mode 100644 index 000000000..975306f8c --- /dev/null +++ b/api/build/auto_cancel.go @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 + +package build + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/token" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/go-vela/types/pipeline" +) + +// AutoCancel is a helper function that checks to see if any pending or running +// builds for the repo can be replaced by the current build. +func AutoCancel(c *gin.Context, b *library.Build, rB *library.Build, r *library.Repo, cancelOpts *pipeline.CancelOptions) (bool, error) { + // if build is the current build, continue + if rB.GetID() == b.GetID() { + return false, nil + } + + // ensure criteria is met before auto canceling (push to same branch, or pull with same action from same head_ref) + if (strings.EqualFold(rB.GetEvent(), constants.EventPush) && + strings.EqualFold(b.GetEvent(), constants.EventPush) && + strings.EqualFold(b.GetBranch(), rB.GetBranch())) || + (strings.EqualFold(rB.GetEvent(), constants.EventPull) && + strings.EqualFold(b.GetEventAction(), rB.GetEventAction()) && + strings.EqualFold(b.GetHeadRef(), rB.GetHeadRef())) { + switch { + case strings.EqualFold(rB.GetStatus(), constants.StatusPending) && cancelOpts.Pending: + // pending build will be handled gracefully by worker once pulled off queue + rB.SetStatus(constants.StatusCanceled) + + _, err := database.FromContext(c).UpdateBuild(c, rB) + if err != nil { + return false, err + } + case strings.EqualFold(rB.GetStatus(), constants.StatusRunning) && cancelOpts.Running: + // call cancelRunning routine for builds already running on worker + err := cancelRunning(c, rB, r) + if err != nil { + return false, err + } + default: + return false, nil + } + + // set error message that references current build + rB.SetError(fmt.Sprintf("build was auto canceled in favor of build %d", b.GetNumber())) + + _, err := database.FromContext(c).UpdateBuild(c, rB) + if err != nil { + // if this call fails, we still canceled the build, so return true + return true, err + } + } + + return true, nil +} + +// cancelRunning is a helper function that determines the executor currently running a build and sends an API call +// to that executor's worker to cancel the build. +func cancelRunning(c *gin.Context, b *library.Build, r *library.Repo) error { + e := new([]library.Executor) + // retrieve the worker + w, err := database.FromContext(c).GetWorkerForHostname(c, b.GetHost()) + if err != nil { + return err + } + + // prepare the request to the worker to retrieve executors + client := http.DefaultClient + client.Timeout = 30 * time.Second + endpoint := fmt.Sprintf("%s/api/v1/executors", w.GetAddress()) + + req, err := http.NewRequestWithContext(context.Background(), "GET", endpoint, nil) + if err != nil { + return err + } + + tm := c.MustGet("token-manager").(*token.Manager) + + // set mint token options + mto := &token.MintTokenOpts{ + Hostname: "vela-server", + TokenType: constants.WorkerAuthTokenType, + TokenDuration: time.Minute * 1, + } + + // mint token + tkn, err := tm.MintToken(mto) + if err != nil { + return err + } + + // add the token to authenticate to the worker + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) + + // make the request to the worker and check the response + resp, err := client.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + // Read Response Body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + // parse response and validate at least one item was returned + err = json.Unmarshal(respBody, e) + if err != nil { + return err + } + + for _, executor := range *e { + // check each executor on the worker running the build to see if it's running the build we want to cancel + if strings.EqualFold(executor.Repo.GetFullName(), r.GetFullName()) && *executor.GetBuild().Number == b.GetNumber() { + // prepare the request to the worker + client := http.DefaultClient + client.Timeout = 30 * time.Second + + // set the API endpoint path we send the request to + u := fmt.Sprintf("%s/api/v1/executors/%d/build/cancel", w.GetAddress(), executor.GetID()) + + req, err := http.NewRequestWithContext(context.Background(), "DELETE", u, nil) + if err != nil { + return err + } + + tm := c.MustGet("token-manager").(*token.Manager) + + // set mint token options + mto := &token.MintTokenOpts{ + Hostname: "vela-server", + TokenType: constants.WorkerAuthTokenType, + TokenDuration: time.Minute * 1, + } + + // mint token + tkn, err := tm.MintToken(mto) + if err != nil { + return err + } + + // add the token to authenticate to the worker + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", tkn)) + + // perform the request to the worker + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read Response Body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + err = json.Unmarshal(respBody, b) + if err != nil { + return err + } + } + } + + return nil +} diff --git a/api/build/cancel.go b/api/build/cancel.go index cbc415024..e8cd19264 100644 --- a/api/build/cancel.go +++ b/api/build/cancel.go @@ -77,7 +77,7 @@ func CancelBuild(c *gin.Context) { e := executors.Retrieve(c) o := org.Retrieve(c) r := repo.Retrieve(c) - u := user.Retrieve(c) + user := user.Retrieve(c) ctx := c.Request.Context() entry := fmt.Sprintf("%s/%d", r.GetFullName(), b.GetNumber()) @@ -89,7 +89,7 @@ func CancelBuild(c *gin.Context) { "build": b.GetNumber(), "org": o, "repo": r.GetName(), - "user": u.GetName(), + "user": user.GetName(), }).Infof("canceling build %s", entry) switch b.GetStatus() { @@ -169,6 +169,16 @@ func CancelBuild(c *gin.Context) { return } + b.SetError(fmt.Sprintf("build was canceled by %s", user.GetName())) + + b, err = database.FromContext(c).UpdateBuild(ctx, b) + if err != nil { + retErr := fmt.Errorf("unable to update status for build %s: %w", entry, err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + c.JSON(resp.StatusCode, b) return diff --git a/api/queue/queue.go b/api/queue/queue.go index 10946e99c..78d4c170b 100644 --- a/api/queue/queue.go +++ b/api/queue/queue.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package queue diff --git a/api/webhook/post.go b/api/webhook/post.go index d3c2fa6c4..aa4449194 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -677,6 +677,40 @@ func PostWebhook(c *gin.Context) { repo, u, ) + + // if anything is provided in the auto_cancel metadata, then we start with true + runAutoCancel := p.Metadata.AutoCancel.Running || p.Metadata.AutoCancel.Pending || p.Metadata.AutoCancel.DefaultBranch + + // if the event is a push to the default branch and the AutoCancel.DefaultBranch value is false, bypass auto cancel + if strings.EqualFold(b.GetEvent(), constants.EventPush) && strings.EqualFold(b.GetBranch(), repo.GetBranch()) && !p.Metadata.AutoCancel.DefaultBranch { + runAutoCancel = false + } + + // if event is push or pull_request:synchronize, there is a chance this build could be superceding a stale build + // + // fetch pending and running builds for this repo in order to validate their merit to continue running. + if runAutoCancel && + ((strings.EqualFold(b.GetEvent(), constants.EventPull) && strings.EqualFold(b.GetEventAction(), constants.ActionSynchronize)) || + strings.EqualFold(b.GetEvent(), constants.EventPush)) { + // fetch pending and running builds + rBs, err := database.FromContext(c).ListPendingAndRunningBuildsForRepo(c, repo) + if err != nil { + logrus.Errorf("unable to fetch pending and running builds for %s: %v", repo.GetFullName(), err) + } + + for _, rB := range rBs { + // call auto cancel routine + canceled, err := build.AutoCancel(c, b, rB, repo, p.Metadata.AutoCancel) + if err != nil { + // continue cancel loop if error, but log based on type of error + if canceled { + logrus.Errorf("unable to update canceled build error message: %v", err) + } else { + logrus.Errorf("unable to cancel running build: %v", err) + } + } + } + } } // handleRepositoryEvent is a helper function that processes repository events from the SCM and updates diff --git a/database/build/interface.go b/database/build/interface.go index 3f790ac10..c2b468e39 100644 --- a/database/build/interface.go +++ b/database/build/interface.go @@ -58,6 +58,8 @@ type BuildInterface interface { ListBuildsForRepo(context.Context, *library.Repo, map[string]interface{}, int64, int64, int, int) ([]*library.Build, int64, error) // ListPendingAndRunningBuilds defines a function that gets a list of pending and running builds. ListPendingAndRunningBuilds(context.Context, string) ([]*library.BuildQueue, error) + // ListPendingAndRunningBuildsForRepo defines a function that gets a list of pending and running builds for a repo. + ListPendingAndRunningBuildsForRepo(context.Context, *library.Repo) ([]*library.Build, error) // UpdateBuild defines a function that updates an existing build. UpdateBuild(context.Context, *library.Build) (*library.Build, error) } diff --git a/database/build/list_pending_running_repo.go b/database/build/list_pending_running_repo.go new file mode 100644 index 000000000..3c82ca419 --- /dev/null +++ b/database/build/list_pending_running_repo.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 + +package build + +import ( + "context" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListPendingAndRunningBuilds gets a list of all pending and running builds in the provided timeframe from the database. +func (e *engine) ListPendingAndRunningBuildsForRepo(ctx context.Context, repo *library.Repo) ([]*library.Build, error) { + e.logger.Trace("listing all pending and running builds from the database") + + // variables to store query results and return value + b := new([]database.Build) + builds := []*library.Build{} + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableBuild). + Select("*"). + Where("repo_id = ?", repo.GetID()). + Where("status = 'running' OR status = 'pending'"). + Find(&b). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, build := range *b { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := build + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Build.ToLibrary + builds = append(builds, tmp.ToLibrary()) + } + + return builds, nil +} diff --git a/database/build/list_pending_running_repo_test.go b/database/build/list_pending_running_repo_test.go new file mode 100644 index 000000000..089a6d7b3 --- /dev/null +++ b/database/build/list_pending_running_repo_test.go @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: Apache-2.0 + +package build + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +func TestBuild_Engine_ListPendingAndRunningBuildsForRepo(t *testing.T) { + // setup types + _buildOne := testBuild() + _buildOne.SetID(1) + _buildOne.SetRepoID(1) + _buildOne.SetNumber(1) + _buildOne.SetStatus("running") + _buildOne.SetCreated(1) + _buildOne.SetDeployPayload(nil) + + _buildTwo := testBuild() + _buildTwo.SetID(2) + _buildTwo.SetRepoID(1) + _buildTwo.SetNumber(2) + _buildTwo.SetStatus("pending") + _buildTwo.SetCreated(1) + _buildTwo.SetDeployPayload(nil) + + _buildThree := testBuild() + _buildThree.SetID(3) + _buildThree.SetRepoID(2) + _buildThree.SetNumber(1) + _buildThree.SetStatus("pending") + _buildThree.SetCreated(1) + _buildThree.SetDeployPayload(nil) + + _repo := testRepo() + _repo.SetID(1) + _repo.SetUserID(1) + _repo.SetHash("baz") + _repo.SetOrg("foo") + _repo.SetName("bar") + _repo.SetFullName("foo/bar") + _repo.SetVisibility("public") + + _repoTwo := testRepo() + _repoTwo.SetID(2) + _repoTwo.SetUserID(1) + _repoTwo.SetHash("bazzy") + _repoTwo.SetOrg("foo") + _repoTwo.SetName("baz") + _repoTwo.SetFullName("foo/baz") + _repoTwo.SetVisibility("public") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected name query result in mock + _rows := sqlmock.NewRows( + []string{"id", "repo_id", "pipeline_id", "number", "parent", "event", "event_action", "status", "error", "enqueued", "created", "started", "finished", "deploy", "deploy_payload", "clone", "source", "title", "message", "commit", "sender", "author", "email", "link", "branch", "ref", "base_ref", "head_ref", "host", "runtime", "distribution", "timestamp"}). + AddRow(2, 1, nil, 2, 0, "", "", "pending", "", 0, 1, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0). + AddRow(1, 1, nil, 1, 0, "", "", "running", "", 0, 1, 0, 0, "", nil, "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", 0) + + // ensure the mock expects the name query + _mock.ExpectQuery(`SELECT * FROM "builds" WHERE repo_id = $1 AND (status = 'running' OR status = 'pending')`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateBuild(context.TODO(), _buildOne) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + _, err = _sqlite.CreateBuild(context.TODO(), _buildTwo) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + _, err = _sqlite.CreateBuild(context.TODO(), _buildThree) + if err != nil { + t.Errorf("unable to create test build for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&database.Repo{}) + if err != nil { + t.Errorf("unable to create repo table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(database.RepoFromLibrary(_repoTwo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Build + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Build{_buildTwo, _buildOne}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Build{_buildOne, _buildTwo}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListPendingAndRunningBuildsForRepo(context.TODO(), _repo) + + if test.failure { + if err == nil { + t.Errorf("ListPendingAndRunningBuildsForRepo for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListPendingAndRunningBuildsForRepo for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListPendingAndRunningBuildsForRepo for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/integration_test.go b/database/integration_test.go index bcf2fcd7f..b30176004 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -296,6 +296,19 @@ func testBuilds(t *testing.T, db Interface, resources *Resources) { } methods["ListBuildsForRepo"] = true + // list the pending / running builds for a repo + list, err = db.ListPendingAndRunningBuildsForRepo(context.TODO(), resources.Repos[0]) + if err != nil { + t.Errorf("unable to list pending and running builds for repo %d: %v", resources.Repos[0].GetID(), err) + } + if int(count) != len(resources.Builds) { + t.Errorf("ListPendingAndRunningBuildsForRepo() is %v, want %v", count, len(resources.Builds)) + } + if !cmp.Equal(list, []*library.Build{resources.Builds[0], resources.Builds[1]}) { + t.Errorf("ListPendingAndRunningBuildsForRepo() is %v, want %v", list, []*library.Build{resources.Builds[0], resources.Builds[1]}) + } + methods["ListPendingAndRunningBuildsForRepo"] = true + // list the pending and running builds queueList, err := db.ListPendingAndRunningBuilds(context.TODO(), "0") if err != nil { diff --git a/database/resource.go b/database/resource.go index 353f25513..35f7c7a97 100644 --- a/database/resource.go +++ b/database/resource.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Ine. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package database diff --git a/database/validate.go b/database/validate.go index 18f07cb84..4c519d081 100644 --- a/database/validate.go +++ b/database/validate.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Ine. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package database diff --git a/database/validate_test.go b/database/validate_test.go index 60761e215..36b60fb98 100644 --- a/database/validate_test.go +++ b/database/validate_test.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Ine. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package database diff --git a/router/middleware/signing_test.go b/router/middleware/signing_test.go index 908c0908b..3f152a3c9 100644 --- a/router/middleware/signing_test.go +++ b/router/middleware/signing_test.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package middleware diff --git a/router/queue.go b/router/queue.go index d3b311b13..ca408ffe3 100644 --- a/router/queue.go +++ b/router/queue.go @@ -1,6 +1,4 @@ -// Copyright (c) 2023 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. +// SPDX-License-Identifier: Apache-2.0 package router From 7370602925b540642ad0f43b0c957ca180dea4b0 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:29:43 -0400 Subject: [PATCH 4/5] fix(api/build): pop executable when pending build is auto canceled (#990) --- api/build/auto_cancel.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/build/auto_cancel.go b/api/build/auto_cancel.go index 975306f8c..9ffb32b26 100644 --- a/api/build/auto_cancel.go +++ b/api/build/auto_cancel.go @@ -43,6 +43,12 @@ func AutoCancel(c *gin.Context, b *library.Build, rB *library.Build, r *library. if err != nil { return false, err } + + // remove executable from table + _, err = database.FromContext(c).PopBuildExecutable(c, rB.GetID()) + if err != nil { + return true, err + } case strings.EqualFold(rB.GetStatus(), constants.StatusRunning) && cancelOpts.Running: // call cancelRunning routine for builds already running on worker err := cancelRunning(c, rB, r) From 4eec5e9b03ab7e7004bfae08fc558eb2b27d87c5 Mon Sep 17 00:00:00 2001 From: claire1618 <55173466+claire1618@users.noreply.github.com> Date: Thu, 26 Oct 2023 13:45:46 -0500 Subject: [PATCH 5/5] chore: updating types for lazy loading secrets (#992) Co-authored-by: Claire.Nicholas --- compiler/native/compile_test.go | 17 +++++++++++++++++ compiler/native/expand_test.go | 9 +++++++++ compiler/native/parse_test.go | 10 ++++++++++ go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 39 insertions(+), 3 deletions(-) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 91948e07b..067349e01 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -231,6 +231,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -238,6 +239,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, }, } @@ -567,6 +569,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -574,6 +577,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, }, } @@ -805,6 +809,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -812,6 +817,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "foo_password", @@ -819,6 +825,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, }, Services: pipeline.ContainerSlice{ @@ -1047,6 +1054,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -1054,6 +1062,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "foo_password", @@ -1061,6 +1070,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, }, Services: pipeline.ContainerSlice{ @@ -2734,6 +2744,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_username", @@ -2741,6 +2752,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "native", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -2748,6 +2760,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "vault", Type: "repo", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_username", @@ -2755,6 +2768,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "native", Type: "org", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -2762,6 +2776,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "vault", Type: "org", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_username", @@ -2769,6 +2784,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "native", Type: "shared", Origin: &pipeline.Container{}, + Pull: "build_start", }, &pipeline.Secret{ Name: "docker_password", @@ -2776,6 +2792,7 @@ func Test_Compile_Inline(t *testing.T) { Engine: "vault", Type: "shared", Origin: &pipeline.Container{}, + Pull: "build_start", }, }, }, diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 808be5a33..0ea845cfc 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -118,6 +118,7 @@ func TestNative_ExpandStages(t *testing.T) { Engine: "native", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Name: "foo_password", @@ -125,6 +126,7 @@ func TestNative_ExpandStages(t *testing.T) { Engine: "vault", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, } @@ -287,6 +289,7 @@ func TestNative_ExpandSteps(t *testing.T) { Engine: "native", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Name: "foo_password", @@ -294,6 +297,7 @@ func TestNative_ExpandSteps(t *testing.T) { Engine: "vault", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, } @@ -506,6 +510,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { Engine: "native", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Name: "foo_password", @@ -513,6 +518,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { Engine: "vault", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Name: "vault_token", @@ -520,6 +526,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { Engine: "native", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Origin: yaml.Origin{ @@ -804,6 +811,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { Engine: "native", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, &yaml.Secret{ Name: "foo_password", @@ -811,6 +819,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { Engine: "vault", Type: "repo", Origin: yaml.Origin{}, + Pull: "build_start", }, } diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index 672ebcaaf..7176438e1 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -320,12 +320,14 @@ func TestNative_Parse_StagesPipeline(t *testing.T) { Key: "org/repo/docker/username", Engine: "native", Type: "repo", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_password", Key: "org/repo/docker/password", Engine: "vault", Type: "repo", + Pull: "build_start", }, }, } @@ -421,12 +423,14 @@ func TestNative_Parse_StepsPipeline(t *testing.T) { Key: "org/repo/docker/username", Engine: "native", Type: "repo", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_password", Key: "org/repo/docker/password", Engine: "vault", Type: "repo", + Pull: "build_start", }, }, } @@ -460,36 +464,42 @@ func TestNative_Parse_Secrets(t *testing.T) { Key: "org/repo/docker/username", Engine: "native", Type: "repo", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_password", Key: "org/repo/docker/password", Engine: "vault", Type: "repo", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_username", Key: "org/docker/username", Engine: "native", Type: "org", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_password", Key: "org/docker/password", Engine: "vault", Type: "org", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_username", Key: "org/team/docker/username", Engine: "native", Type: "shared", + Pull: "build_start", }, &yaml.Secret{ Name: "docker_password", Key: "org/team/docker/password", Engine: "vault", Type: "shared", + Pull: "build_start", }, }, } diff --git a/go.mod b/go.mod index 1aa96d380..d0cf1c469 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 github.com/go-playground/assert/v2 v2.2.0 - github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af + github.com/go-vela/types v0.21.1-0.20231024201126-19101a5b1346 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/go-cmp v0.5.9 github.com/google/go-github/v55 v55.0.0 diff --git a/go.sum b/go.sum index ccc23bbaa..fa1e03179 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af h1:qiP6pXFDyPDDP+hy8zY+nhmoWv9aoQrrnNmfAAT6yCA= -github.com/go-vela/types v0.21.1-0.20231012142227-0c0b890487af/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ= +github.com/go-vela/types v0.21.1-0.20231024201126-19101a5b1346 h1:8VqRJ02KcAxV+gHuxLzuPuNaf7EOE/zfBomEV+UPj/E= +github.com/go-vela/types v0.21.1-0.20231024201126-19101a5b1346/go.mod h1:Jn8K28uj7mACc55fkFgaIzL0q45iXydOFGEeoSeHUtQ= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=