From 38cd71424d3975d1a35fcf4e722b3f406ae03584 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 6 May 2024 10:45:31 -0400 Subject: [PATCH 01/14] enhance(mock): add dashboard responses from server in mock (#1118) * init commit * add to server * proper dash card resp in mock * fix copypasta --- mock/server/dashboard.go | 252 ++++++++++++++++++++++++++++++++++ mock/server/dashboard_test.go | 59 ++++++++ mock/server/server.go | 6 + 3 files changed, 317 insertions(+) create mode 100644 mock/server/dashboard.go create mode 100644 mock/server/dashboard_test.go diff --git a/mock/server/dashboard.go b/mock/server/dashboard.go new file mode 100644 index 000000000..bd560d7c2 --- /dev/null +++ b/mock/server/dashboard.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/types" +) + +const ( + // DashboardResp represents a JSON return for a dashboard. + DashboardResp = `{ + "id": "c976470d-34c1-49b2-9a98-1035871c576b", + "name": "my-dashboard", + "created_at": 1714573212, + "created_by": "Octocat", + "updated_at": 1714573212, + "updated_by": "Octocat", + "admins": [ + { + "id": 1, + "name": "Octocat", + "active": true + } + ], + "repos": [ + { + "id": 1, + "name": "Octocat/vela-repo", + "branches": [ + "main" + ], + "events": [ + "push" + ] + } + ] +}` + + // DashCardResp represents a JSON return for a DashCard. + DashCardResp = `{ + "dashboard": { + "id": "6e9f84c3-d853-4afb-b56e-99ff200264c0", + "name": "dashboard-1", + "created_at": 1714677999, + "created_by": "Octocat", + "updated_at": 1714678173, + "updated_by": "Octocat", + "admins": [ + { + "id": 1, + "name": "Octocat", + "active": true + } + ], + "repos": [ + { + "id": 2, + "name": "Octocat/test-repo" + }, + { + "id": 1, + "name": "Octocat/test-repo-2" + } + ] + }, + "repos": [ + { + "org": "Octocat", + "name": "test-repo", + "counter": 1, + "builds": [ + { + "number": 1, + "started": 1714678666, + "finished": 1714678672, + "sender": "Octocat", + "status": "failure", + "event": "deployment", + "branch": "refs/heads/main", + "link": "http://vela/Octocat/test-repo/1" + } + ] + }, + { + "org": "Octocat", + "name": "test-repo-2" + } + ] +}` + + // DashCardsResp represents a JSON return for multiple DashCards. + DashCardsResp = `[ +{ + "dashboard": { + "id": "6e9f84c3-d853-4afb-b56e-99ff200264c0", + "name": "dashboard-1", + "created_at": 1714677999, + "created_by": "Octocat", + "updated_at": 1714678173, + "updated_by": "Octocat", + "admins": [ + { + "id": 1, + "name": "Octocat", + "active": true + } + ], + "repos": [ + { + "id": 2, + "name": "Octocat/test-repo" + }, + { + "id": 1, + "name": "Octocat/test-repo-2" + } + ] + }, + "repos": [ + { + "org": "Octocat", + "name": "test-repo", + "counter": 1, + "builds": [ + { + "number": 1, + "started": 1714678666, + "finished": 1714678672, + "sender": "Octocat", + "status": "failure", + "event": "deployment", + "branch": "refs/heads/main", + "link": "http://vela/Octocat/test-repo/1" + } + ] + }, + { + "org": "Octocat", + "name": "test-repo-2" + } + ] +}, +{ + "dashboard": { + "id": "6e9f84c3-d853-4afb-b56e-99ff200264c1", + "name": "dashboard-2", + "created_at": 1714677999, + "created_by": "Octocat", + "updated_at": 1714678173, + "updated_by": "Octocat", + "admins": [ + { + "id": 1, + "name": "Octocat", + "active": true + } + ], + "repos": [ + { + "id": 2, + "name": "Octocat/test-repo" + }, + { + "id": 1, + "name": "Octocat/test-repo-2" + } + ] + }, + "repos": [ + { + "org": "Octocat", + "name": "test-repo", + "counter": 1, + "builds": [ + { + "number": 1, + "started": 1714678666, + "finished": 1714678672, + "sender": "Octocat", + "status": "failure", + "event": "deployment", + "branch": "refs/heads/main", + "link": "http://vela/Octocat/test-repo/1" + } + ] + }, + { + "org": "Octocat", + "name": "test-repo-2" + } + ] +} +]` +) + +// getDashboards returns mock JSON for a http GET. +func getDashboards(c *gin.Context) { + data := []byte(DashCardsResp) + + var body []api.Dashboard + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// getDashboard has a param :dashboard returns mock JSON for a http GET. +func getDashboard(c *gin.Context) { + d := c.Param("dashboard") + + if strings.EqualFold(d, "0") { + msg := fmt.Sprintf("Dashboard %s does not exist", d) + + c.AbortWithStatusJSON(http.StatusNotFound, types.Error{Message: &msg}) + + return + } + + data := []byte(DashCardResp) + + var body api.Dashboard + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// addDashboard returns mock JSON for a http POST. +func addDashboard(c *gin.Context) { + data := []byte(DashboardResp) + + var body api.Dashboard + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusCreated, body) +} + +// updateDashboard returns mock JSON for a http PUT. +func updateDashboard(c *gin.Context) { + data := []byte(DashboardResp) + + var body api.Dashboard + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} diff --git a/mock/server/dashboard_test.go b/mock/server/dashboard_test.go new file mode 100644 index 000000000..de09a0257 --- /dev/null +++ b/mock/server/dashboard_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types" +) + +func TestDashboard_ActiveDashboardResp(t *testing.T) { + testDashboard := api.Dashboard{} + + err := json.Unmarshal([]byte(DashboardResp), &testDashboard) + if err != nil { + t.Errorf("error unmarshaling dashboard: %v", err) + } + + tDashboard := reflect.TypeOf(testDashboard) + + for i := 0; i < tDashboard.NumField(); i++ { + if reflect.ValueOf(testDashboard).Field(i).IsNil() { + t.Errorf("DashboardResp missing field %s", tDashboard.Field(i).Name) + } + } + + testDashCard := api.DashCard{} + + err = json.Unmarshal([]byte(DashCardResp), &testDashCard) + if err != nil { + t.Errorf("error unmarshaling dash card: %v", err) + } + + tDashCard := reflect.TypeOf(testDashCard) + + for i := 0; i < tDashCard.NumField(); i++ { + if reflect.ValueOf(testDashCard).Field(i).IsNil() { + t.Errorf("DashCardResp missing field %s", tDashCard.Field(i).Name) + } + } + + testDashCards := []api.DashCard{} + err = json.Unmarshal([]byte(DashCardsResp), &testDashCards) + if err != nil { + t.Errorf("error unmarshaling dash cards: %v", err) + } + + for _, testDashCard := range testDashCards { + tDashCard := reflect.TypeOf(testDashCard) + + for i := 0; i < tDashCard.NumField(); i++ { + if reflect.ValueOf(testDashCard).Field(i).IsNil() { + t.Errorf("DashboardsResp missing field %s", tDashboard.Field(i).Name) + } + } + } +} diff --git a/mock/server/server.go b/mock/server/server.go index 0b7dcff79..fb6161ccd 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -43,6 +43,12 @@ func FakeHandler() http.Handler { e.GET("/api/v1/repos/:org/:repo/builds/:build/token", buildToken) e.GET("/api/v1/repos/:org/:repo/builds/:build/executable", buildExecutable) + // mock endpoints for dashboard calls + e.GET("/api/v1/dashboards/:dashboard", getDashboard) + e.GET("/api/v1/user/dashboards", getDashboards) + e.POST("/api/v1/dashboards", addDashboard) + e.PUT("/api/v1/dashboards/:dashboard", updateDashboard) + // mock endpoints for deployment calls e.GET("/api/v1/deployments/:org/:repo", getDeployments) e.POST("/api/v1/deployments/:org/:repo", addDeployment) From 993a937238015bc04fe5fb1c2e4d29f2fd8f940d Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 6 May 2024 10:48:26 -0400 Subject: [PATCH 02/14] fix(dashboards): add active status on repo partial (#1119) --- api/dashboard/get.go | 1 + api/types/dashboard.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/dashboard/get.go b/api/dashboard/get.go index f36f92ee4..da4239667 100644 --- a/api/dashboard/get.go +++ b/api/dashboard/get.go @@ -98,6 +98,7 @@ func buildRepoPartials(c context.Context, repos []*types.DashboardRepo) ([]types repo.Org = dbRepo.GetOrg() repo.Name = dbRepo.GetName() repo.Counter = dbRepo.GetCounter() + repo.Active = dbRepo.GetActive() // list last 5 builds for repo given the branch and event filters builds, err := database.FromContext(c).ListBuildsForDashboardRepo(c, dbRepo, r.GetBranches(), r.GetEvents()) diff --git a/api/types/dashboard.go b/api/types/dashboard.go index 053942daf..11c3883ee 100644 --- a/api/types/dashboard.go +++ b/api/types/dashboard.go @@ -12,6 +12,7 @@ type RepoPartial struct { Org string `json:"org,omitempty"` Name string `json:"name,omitempty"` Counter int `json:"counter,omitempty"` + Active bool `json:"active,omitempty"` Builds []BuildPartial `json:"builds,omitempty"` } From 8ad123451469039e0125af1943e916c211415642 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 6 May 2024 11:41:18 -0400 Subject: [PATCH 03/14] fix(mock): correct type for mock response (#1121) --- mock/server/dashboard.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mock/server/dashboard.go b/mock/server/dashboard.go index bd560d7c2..c2a017050 100644 --- a/mock/server/dashboard.go +++ b/mock/server/dashboard.go @@ -205,7 +205,7 @@ const ( func getDashboards(c *gin.Context) { data := []byte(DashCardsResp) - var body []api.Dashboard + var body []api.DashCard _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -225,7 +225,7 @@ func getDashboard(c *gin.Context) { data := []byte(DashCardResp) - var body api.Dashboard + var body api.DashCard _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) From 7f1a4c0aa6a36030eea51386cd22452ee615473f Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Tue, 7 May 2024 15:43:21 -0400 Subject: [PATCH 04/14] fix(ci): use intermediate env var for PR title (#1122) --- .github/workflows/validate-pr-title.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-pr-title.yml b/.github/workflows/validate-pr-title.yml index ec3674f79..84ffec5ac 100644 --- a/.github/workflows/validate-pr-title.yml +++ b/.github/workflows/validate-pr-title.yml @@ -13,5 +13,7 @@ jobs: steps: - name: validate title + env: + TITLE: ${{ github.event.pull_request.title }} run: | - echo "${{ github.event.pull_request.title }}" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|)!?:\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) + echo "$TITLE" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|)!?:\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) From 300ca456e3bbbf1bc8167dbae0db177370be690b Mon Sep 17 00:00:00 2001 From: Win San Date: Wed, 8 May 2024 08:37:21 -0500 Subject: [PATCH 05/14] feat(schedule)!: show schedule errors and nest object (#1108) * Update schedule error logging * Migrate schedule from types to server * Update schedule function names & comments * Turn Schedule into a nested object * Populate owner in Schedule - Preload owner in Repo, which is nested in Schedule - Update unit tests to reflect change - Update integration tests to reflect change * Update tests to reflect nested Schedule * Run lintfix * Fix naming and string formatting * Add check before clearing Schedule error --------- Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Co-authored-by: david may <1301201+wass3r@users.noreply.github.com> --- api/admin/build.go | 2 +- api/build/approve.go | 2 +- api/schedule/create.go | 8 +- api/schedule/update.go | 4 +- api/types/schedule.go | 368 +++++++++++++++++++++ api/types/schedule_test.go | 232 +++++++++++++ cmd/vela-server/schedule.go | 45 ++- database/build/opts.go | 4 +- database/integration_test.go | 70 ++-- database/resource.go | 1 + database/schedule/count_active_test.go | 72 +++- database/schedule/count_repo_test.go | 84 +++-- database/schedule/count_test.go | 74 ++++- database/schedule/create.go | 21 +- database/schedule/create_test.go | 65 +++- database/schedule/delete.go | 10 +- database/schedule/delete_test.go | 53 ++- database/schedule/get.go | 18 +- database/schedule/get_repo.go | 17 +- database/schedule/get_repo_test.go | 100 +++++- database/schedule/get_test.go | 96 +++++- database/schedule/interface.go | 17 +- database/schedule/list.go | 20 +- database/schedule/list_active.go | 20 +- database/schedule/list_active_test.go | 114 +++++-- database/schedule/list_repo.go | 21 +- database/schedule/list_repo_test.go | 120 +++++-- database/schedule/list_test.go | 117 +++++-- database/schedule/opts.go | 10 + database/schedule/schedule.go | 2 + database/schedule/schedule_test.go | 43 --- database/schedule/table.go | 2 + database/schedule/update.go | 20 +- database/schedule/update_test.go | 129 ++++++-- database/testutils/api_resources.go | 17 + database/types/repo_test.go | 2 +- database/types/schedule.go | 155 +++++++++ database/types/schedule_test.go | 234 +++++++++++++ mock/server/schedule.go | 182 ++++++++-- mock/server/schedule_test.go | 4 +- router/middleware/schedule/context.go | 8 +- router/middleware/schedule/context_test.go | 6 +- router/middleware/schedule/schedule.go | 4 +- 43 files changed, 2219 insertions(+), 374 deletions(-) create mode 100644 api/types/schedule.go create mode 100644 api/types/schedule_test.go create mode 100644 database/types/schedule.go create mode 100644 database/types/schedule_test.go diff --git a/api/admin/build.go b/api/admin/build.go index 5e8d06694..1a4fe0489 100644 --- a/api/admin/build.go +++ b/api/admin/build.go @@ -45,7 +45,7 @@ import ( // "$ref": "#/definitions/Error" // AllBuildsQueue represents the API handler to -// captures all running and pending builds stored in the database. +// capture all running and pending builds stored in the database. func AllBuildsQueue(c *gin.Context) { // capture middleware values ctx := c.Request.Context() diff --git a/api/build/approve.go b/api/build/approve.go index cd022bd79..16c1fcc65 100644 --- a/api/build/approve.go +++ b/api/build/approve.go @@ -70,7 +70,7 @@ import ( // schema: // "$ref": "#/definitions/Error" -// CreateBuild represents the API handler to approve a build to run in the configured backend. +// ApproveBuild represents the API handler to approve a build to run in the configured backend. func ApproveBuild(c *gin.Context) { // capture middleware values b := build.Retrieve(c) diff --git a/api/schedule/create.go b/api/schedule/create.go index 7a9439b8c..a691230e6 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -11,11 +11,11 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" ) // swagger:operation POST /api/v1/schedules/{org}/{repo} schedules CreateSchedule @@ -81,7 +81,7 @@ func CreateSchedule(c *gin.Context) { minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) // capture body from API request - input := new(library.Schedule) + input := new(api.Schedule) err := c.Bind(input) if err != nil { @@ -127,11 +127,11 @@ func CreateSchedule(c *gin.Context) { return } - s := new(library.Schedule) + s := new(api.Schedule) // update fields in schedule object s.SetCreatedBy(u.GetName()) - s.SetRepoID(r.GetID()) + s.SetRepo(r) s.SetName(input.GetName()) s.SetEntry(input.GetEntry()) s.SetCreatedAt(time.Now().UTC().Unix()) diff --git a/api/schedule/update.go b/api/schedule/update.go index 41da289ec..c29678570 100644 --- a/api/schedule/update.go +++ b/api/schedule/update.go @@ -10,12 +10,12 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/schedule" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" ) // swagger:operation PUT /api/v1/schedules/{org}/{repo}/{schedule} schedules UpdateSchedule @@ -88,7 +88,7 @@ func UpdateSchedule(c *gin.Context) { }).Infof("updating schedule %s", scheduleName) // capture body from API request - input := new(library.Schedule) + input := new(api.Schedule) err := c.Bind(input) if err != nil { diff --git a/api/types/schedule.go b/api/types/schedule.go new file mode 100644 index 000000000..c21568a27 --- /dev/null +++ b/api/types/schedule.go @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" +) + +// Schedule is the API representation of a schedule for a repo. +// +// swagger:model Schedule +type Schedule struct { + ID *int64 `json:"id,omitempty"` + Repo *Repo `json:"repo,omitempty"` + Active *bool `json:"active,omitempty"` + Name *string `json:"name,omitempty"` + Entry *string `json:"entry,omitempty"` + CreatedAt *int64 `json:"created_at,omitempty"` + CreatedBy *string `json:"created_by,omitempty"` + UpdatedAt *int64 `json:"updated_at,omitempty"` + UpdatedBy *string `json:"updated_by,omitempty"` + ScheduledAt *int64 `json:"scheduled_at,omitempty"` + Branch *string `json:"branch,omitempty"` + Error *string `json:"error,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetID() int64 { + // return zero value if Schedule type or ID field is nil + if s == nil || s.ID == nil { + return 0 + } + + return *s.ID +} + +// GetRepo returns the Repo field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetRepo() *Repo { + // return zero value if Schedule type or RepoID field is nil + if s == nil || s.Repo == nil { + return new(Repo) + } + + return s.Repo +} + +// GetActive returns the Active field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetActive() bool { + // return zero value if Schedule type or Active field is nil + if s == nil || s.Active == nil { + return false + } + + return *s.Active +} + +// GetName returns the Name field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetName() string { + // return zero value if Schedule type or Name field is nil + if s == nil || s.Name == nil { + return "" + } + + return *s.Name +} + +// GetEntry returns the Entry field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetEntry() string { + // return zero value if Schedule type or Entry field is nil + if s == nil || s.Entry == nil { + return "" + } + + return *s.Entry +} + +// GetCreatedAt returns the CreatedAt field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetCreatedAt() int64 { + // return zero value if Schedule type or CreatedAt field is nil + if s == nil || s.CreatedAt == nil { + return 0 + } + + return *s.CreatedAt +} + +// GetCreatedBy returns the CreatedBy field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetCreatedBy() string { + // return zero value if Schedule type or CreatedBy field is nil + if s == nil || s.CreatedBy == nil { + return "" + } + + return *s.CreatedBy +} + +// GetUpdatedAt returns the UpdatedAt field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetUpdatedAt() int64 { + // return zero value if Schedule type or UpdatedAt field is nil + if s == nil || s.UpdatedAt == nil { + return 0 + } + + return *s.UpdatedAt +} + +// GetUpdatedBy returns the UpdatedBy field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetUpdatedBy() string { + // return zero value if Schedule type or UpdatedBy field is nil + if s == nil || s.UpdatedBy == nil { + return "" + } + + return *s.UpdatedBy +} + +// GetScheduledAt returns the ScheduledAt field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetScheduledAt() int64 { + // return zero value if Schedule type or ScheduledAt field is nil + if s == nil || s.ScheduledAt == nil { + return 0 + } + + return *s.ScheduledAt +} + +// GetBranch returns the Branch field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetBranch() string { + // return zero value if Schedule type or ScheduledAt field is nil + if s == nil || s.Branch == nil { + return "" + } + + return *s.Branch +} + +// GetError returns the Error field. +// +// When the provided Schedule type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (s *Schedule) GetError() string { + // return zero value if Schedule type or Error field is nil + if s == nil || s.Error == nil { + return "" + } + + return *s.Error +} + +// SetID sets the ID field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetID(id int64) { + // return if Schedule type is nil + if s == nil { + return + } + + s.ID = &id +} + +// SetRepo sets the Repo field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetRepo(v *Repo) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Repo = v +} + +// SetActive sets the Active field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetActive(active bool) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Active = &active +} + +// SetName sets the Name field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetName(name string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Name = &name +} + +// SetEntry sets the Entry field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetEntry(entry string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Entry = &entry +} + +// SetCreatedAt sets the CreatedAt field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetCreatedAt(createdAt int64) { + // return if Schedule type is nil + if s == nil { + return + } + + s.CreatedAt = &createdAt +} + +// SetCreatedBy sets the CreatedBy field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetCreatedBy(createdBy string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.CreatedBy = &createdBy +} + +// SetUpdatedAt sets the UpdatedAt field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetUpdatedAt(updatedAt int64) { + // return if Schedule type is nil + if s == nil { + return + } + + s.UpdatedAt = &updatedAt +} + +// SetUpdatedBy sets the UpdatedBy field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetUpdatedBy(updatedBy string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.UpdatedBy = &updatedBy +} + +// SetScheduledAt sets the ScheduledAt field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetScheduledAt(scheduledAt int64) { + // return if Schedule type is nil + if s == nil { + return + } + + s.ScheduledAt = &scheduledAt +} + +// SetBranch sets the Branch field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetBranch(branch string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Branch = &branch +} + +// SetError sets the Error field. +// +// When the provided Schedule type is nil, it +// will set nothing and immediately return. +func (s *Schedule) SetError(err string) { + // return if Schedule type is nil + if s == nil { + return + } + + s.Error = &err +} + +// String implements the Stringer interface for the Schedule type. +func (s *Schedule) String() string { + return fmt.Sprintf(`{ + Active: %t, + CreatedAt: %d, + CreatedBy: %s, + Entry: %s, + ID: %d, + Name: %s, + Repo: %v, + ScheduledAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, + Branch: %s, + Error: %s, +}`, + s.GetActive(), + s.GetCreatedAt(), + s.GetCreatedBy(), + s.GetEntry(), + s.GetID(), + s.GetName(), + s.GetRepo(), + s.GetScheduledAt(), + s.GetUpdatedAt(), + s.GetUpdatedBy(), + s.GetBranch(), + s.GetError(), + ) +} diff --git a/api/types/schedule_test.go b/api/types/schedule_test.go new file mode 100644 index 000000000..ca5bb09db --- /dev/null +++ b/api/types/schedule_test.go @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestTypes_Schedule_Getters(t *testing.T) { + tests := []struct { + name string + schedule *Schedule + want *Schedule + }{ + { + name: "schedule with fields", + schedule: testSchedule(), + want: testSchedule(), + }, + { + name: "schedule with empty fields", + schedule: new(Schedule), + want: new(Schedule), + }, + { + name: "empty schedule", + schedule: nil, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.schedule.GetID() != test.want.GetID() { + t.Errorf("GetID is %v, want %v", test.schedule.GetID(), test.want.GetID()) + } + + if test.schedule.GetRepo().GetID() != test.want.GetRepo().GetID() { + t.Errorf("GetRepoID is %v, want %v", test.schedule.GetRepo().GetID(), test.want.GetRepo().GetID()) + } + + if test.schedule.GetActive() != test.want.GetActive() { + t.Errorf("GetActive is %v, want %v", test.schedule.GetActive(), test.want.GetActive()) + } + + if test.schedule.GetName() != test.want.GetName() { + t.Errorf("GetName is %v, want %v", test.schedule.GetName(), test.want.GetName()) + } + + if test.schedule.GetEntry() != test.want.GetEntry() { + t.Errorf("GetEntry is %v, want %v", test.schedule.GetEntry(), test.want.GetEntry()) + } + + if test.schedule.GetCreatedAt() != test.want.GetCreatedAt() { + t.Errorf("GetCreatedAt is %v, want %v", test.schedule.GetCreatedAt(), test.want.GetCreatedAt()) + } + + if test.schedule.GetCreatedBy() != test.want.GetCreatedBy() { + t.Errorf("GetCreatedBy is %v, want %v", test.schedule.GetCreatedBy(), test.want.GetCreatedBy()) + } + + if test.schedule.GetUpdatedAt() != test.want.GetUpdatedAt() { + t.Errorf("GetUpdatedAt is %v, want %v", test.schedule.GetUpdatedAt(), test.want.GetUpdatedAt()) + } + + if test.schedule.GetUpdatedBy() != test.want.GetUpdatedBy() { + t.Errorf("GetUpdatedBy is %v, want %v", test.schedule.GetUpdatedBy(), test.want.GetUpdatedBy()) + } + + if test.schedule.GetScheduledAt() != test.want.GetScheduledAt() { + t.Errorf("GetScheduledAt is %v, want %v", test.schedule.GetScheduledAt(), test.want.GetScheduledAt()) + } + + if test.schedule.GetBranch() != test.want.GetBranch() { + t.Errorf("GetBranch is %v, want %v", test.schedule.GetBranch(), test.want.GetBranch()) + } + + if test.schedule.GetError() != test.want.GetError() { + t.Errorf("GetError is %v, want %v", test.schedule.GetError(), test.want.GetError()) + } + }) + } +} + +func TestTypes_Schedule_Setters(t *testing.T) { + tests := []struct { + name string + schedule *Schedule + want *Schedule + }{ + { + name: "schedule with fields", + schedule: testSchedule(), + want: testSchedule(), + }, + { + name: "schedule with empty fields", + schedule: new(Schedule), + want: new(Schedule), + }, + { + name: "empty schedule", + schedule: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.schedule.SetID(test.want.GetID()) + test.schedule.SetRepo(test.want.GetRepo()) + test.schedule.SetActive(test.want.GetActive()) + test.schedule.SetName(test.want.GetName()) + test.schedule.SetEntry(test.want.GetEntry()) + test.schedule.SetCreatedAt(test.want.GetCreatedAt()) + test.schedule.SetCreatedBy(test.want.GetCreatedBy()) + test.schedule.SetUpdatedAt(test.want.GetUpdatedAt()) + test.schedule.SetUpdatedBy(test.want.GetUpdatedBy()) + test.schedule.SetScheduledAt(test.want.GetScheduledAt()) + test.schedule.SetBranch(test.want.GetBranch()) + test.schedule.SetError(test.want.GetError()) + + if test.schedule.GetID() != test.want.GetID() { + t.Errorf("SetID is %v, want %v", test.schedule.GetID(), test.want.GetID()) + } + + if test.schedule.GetRepo().GetID() != test.want.GetRepo().GetID() { + t.Errorf("SetRepoID is %v, want %v", test.schedule.GetRepo().GetID(), test.want.GetRepo().GetID()) + } + + if test.schedule.GetActive() != test.want.GetActive() { + t.Errorf("SetActive is %v, want %v", test.schedule.GetActive(), test.want.GetActive()) + } + + if test.schedule.GetName() != test.want.GetName() { + t.Errorf("SetName is %v, want %v", test.schedule.GetName(), test.want.GetName()) + } + + if test.schedule.GetEntry() != test.want.GetEntry() { + t.Errorf("SetEntry is %v, want %v", test.schedule.GetEntry(), test.want.GetEntry()) + } + + if test.schedule.GetCreatedAt() != test.want.GetCreatedAt() { + t.Errorf("SetCreatedAt is %v, want %v", test.schedule.GetCreatedAt(), test.want.GetCreatedAt()) + } + + if test.schedule.GetCreatedBy() != test.want.GetCreatedBy() { + t.Errorf("SetCreatedBy is %v, want %v", test.schedule.GetCreatedBy(), test.want.GetCreatedBy()) + } + + if test.schedule.GetUpdatedAt() != test.want.GetUpdatedAt() { + t.Errorf("SetUpdatedAt is %v, want %v", test.schedule.GetUpdatedAt(), test.want.GetUpdatedAt()) + } + + if test.schedule.GetUpdatedBy() != test.want.GetUpdatedBy() { + t.Errorf("SetUpdatedBy is %v, want %v", test.schedule.GetUpdatedBy(), test.want.GetUpdatedBy()) + } + + if test.schedule.GetScheduledAt() != test.want.GetScheduledAt() { + t.Errorf("SetScheduledAt is %v, want %v", test.schedule.GetScheduledAt(), test.want.GetScheduledAt()) + } + + if test.schedule.GetBranch() != test.want.GetBranch() { + t.Errorf("SetBranch is %v, want %v", test.schedule.GetBranch(), test.want.GetBranch()) + } + + if test.schedule.GetError() != test.want.GetError() { + t.Errorf("SetError is %v, want %v", test.schedule.GetError(), test.want.GetError()) + } + }) + } +} + +func TestTypes_Schedule_String(t *testing.T) { + s := testSchedule() + + want := fmt.Sprintf(`{ + Active: %t, + CreatedAt: %d, + CreatedBy: %s, + Entry: %s, + ID: %d, + Name: %s, + Repo: %v, + ScheduledAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, + Branch: %s, + Error: %s, +}`, + s.GetActive(), + s.GetCreatedAt(), + s.GetCreatedBy(), + s.GetEntry(), + s.GetID(), + s.GetName(), + s.GetRepo(), + s.GetScheduledAt(), + s.GetUpdatedAt(), + s.GetUpdatedBy(), + s.GetBranch(), + s.GetError(), + ) + + got := s.String() + if !strings.EqualFold(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testSchedule is a test helper function to create a Schedule type with all fields set to a fake value. +func testSchedule() *Schedule { + s := new(Schedule) + s.SetID(1) + s.SetRepo(testRepo()) + s.SetActive(true) + s.SetName("nightly") + s.SetEntry("0 0 * * *") + s.SetCreatedAt(time.Now().UTC().Unix()) + s.SetCreatedBy("user1") + s.SetUpdatedAt(time.Now().Add(time.Hour * 1).UTC().Unix()) + s.SetUpdatedBy("user2") + s.SetScheduledAt(time.Now().Add(time.Hour * 2).UTC().Unix()) + s.SetBranch("main") + s.SetError("unable to trigger build for schedule nightly: unknown character") + + return s +} diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index ee4a32acd..b86e70b79 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -21,7 +21,6 @@ import ( "github.com/go-vela/server/scm" "github.com/go-vela/server/util" "github.com/go-vela/types/constants" - "github.com/go-vela/types/library" ) const ( @@ -55,7 +54,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En // amount of time to get to the end of the list. schedule, err := database.GetSchedule(ctx, s.GetID()) if err != nil { - logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + handleError(ctx, database, err, schedule) continue } @@ -75,7 +74,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En // i.e. if it's 4:02 on five minute intervals, this will be 4:00 prevTime, err := gronx.PrevTick(schedule.GetEntry(), true) if err != nil { - logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + handleError(ctx, database, err, schedule) continue } @@ -85,7 +84,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En // i.e. if it's 4:02 on five minute intervals, this will be 4:05 nextTime, err := gronx.NextTickAfter(schedule.GetEntry(), scheduled, true) if err != nil { - logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + handleError(ctx, database, err, schedule) continue } @@ -117,7 +116,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En // send API call to update schedule for ensuring scheduled_at field is set _, err = database.UpdateSchedule(ctx, schedule, false) if err != nil { - logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + handleError(ctx, database, err, schedule) continue } @@ -125,19 +124,32 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En // process the schedule and trigger a new build err = processSchedule(ctx, schedule, compiler, database, metadata, queue, scm, allowList) if err != nil { - logrus.WithError(err).Warnf("%s %s", scheduleErr, schedule.GetName()) + handleError(ctx, database, err, schedule) continue } + + // successfully scheduled build so clear error message, if not already cleared + if schedule.GetError() != "" { + schedule.SetError("") + + // send API call to update schedule with the error message field cleared + _, err = database.UpdateSchedule(ctx, schedule, true) + if err != nil { + handleError(ctx, database, err, schedule) + + continue + } + } } return nil } // processSchedule will, given a schedule, process it and trigger a new build. -func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedule(ctx context.Context, s *api.Schedule, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { // send API call to capture the repo for the schedule - r, err := database.GetRepo(ctx, s.GetRepoID()) + r, err := database.GetRepo(ctx, s.GetRepo().GetID()) if err != nil { return fmt.Errorf("unable to fetch repo: %w", err) } @@ -203,3 +215,20 @@ func processSchedule(ctx context.Context, s *library.Schedule, compiler compiler return nil } + +func handleError(ctx context.Context, database database.Interface, err error, schedule *api.Schedule) { + // log the error message + logrus.WithError(err).Warnf("%s %s: %s", scheduleErr, schedule.GetName(), err.Error()) + + // format the error message + msg := fmt.Sprintf("%s %s: %s", scheduleErr, schedule.GetName(), err.Error()) + + // update the message field with the error message + schedule.SetError(msg) + + // send API call to update schedule to ensure message field is set + _, err = database.UpdateSchedule(ctx, schedule, true) + if err != nil { + logrus.WithError(err).Warnf("%s %s: %s", scheduleErr, schedule.GetName(), err.Error()) + } +} diff --git a/database/build/opts.go b/database/build/opts.go index 0c3a661be..a83ef0a16 100644 --- a/database/build/opts.go +++ b/database/build/opts.go @@ -22,10 +22,10 @@ func WithClient(client *gorm.DB) EngineOpt { } } -// WithEncryptionKey sets the encryption key in the database engine for Repos. +// WithEncryptionKey sets the encryption key in the database engine for Builds. func WithEncryptionKey(key string) EngineOpt { return func(e *engine) error { - // set the encryption key in the repo engine + // set the encryption key in the build engine e.config.EncryptionKey = key return nil diff --git a/database/integration_test.go b/database/integration_test.go index 36e2cb5a6..4f28bbb4e 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -43,7 +43,7 @@ type Resources struct { Logs []*library.Log Pipelines []*library.Pipeline Repos []*api.Repo - Schedules []*library.Schedule + Schedules []*api.Schedule Secrets []*library.Secret Services []*library.Service Steps []*library.Step @@ -1278,11 +1278,25 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods[element.Method(i).Name] = false } - ctx := context.TODO() + // create owners + for _, user := range resources.Users { + _, err := db.CreateUser(context.TODO(), user) + if err != nil { + t.Errorf("unable to create user %d: %v", user.GetID(), err) + } + } + + // create the repos + for _, repo := range resources.Repos { + _, err := db.CreateRepo(context.TODO(), repo) + if err != nil { + t.Errorf("unable to create repo %d: %v", repo.GetID(), err) + } + } // create the schedules for _, schedule := range resources.Schedules { - _, err := db.CreateSchedule(ctx, schedule) + _, err := db.CreateSchedule(context.TODO(), schedule) if err != nil { t.Errorf("unable to create schedule %d: %v", schedule.GetID(), err) } @@ -1290,7 +1304,7 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods["CreateSchedule"] = true // count the schedules - count, err := db.CountSchedules(ctx) + count, err := db.CountSchedules(context.TODO()) if err != nil { t.Errorf("unable to count schedules: %v", err) } @@ -1300,7 +1314,7 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods["CountSchedules"] = true // count the schedules for a repo - count, err = db.CountSchedulesForRepo(ctx, resources.Repos[0]) + count, err = db.CountSchedulesForRepo(context.TODO(), resources.Repos[0]) if err != nil { t.Errorf("unable to count schedules for repo %d: %v", resources.Repos[0].GetID(), err) } @@ -1310,7 +1324,7 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods["CountSchedulesForRepo"] = true // list the schedules - list, err := db.ListSchedules(ctx) + list, err := db.ListSchedules(context.TODO()) if err != nil { t.Errorf("unable to list schedules: %v", err) } @@ -1320,7 +1334,7 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods["ListSchedules"] = true // list the active schedules - list, err = db.ListActiveSchedules(ctx) + list, err = db.ListActiveSchedules(context.TODO()) if err != nil { t.Errorf("unable to list schedules: %v", err) } @@ -1330,22 +1344,22 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { methods["ListActiveSchedules"] = true // list the schedules for a repo - list, count, err = db.ListSchedulesForRepo(ctx, resources.Repos[0], 1, 10) + list, count, err = db.ListSchedulesForRepo(context.TODO(), resources.Repos[0], 1, 10) if err != nil { t.Errorf("unable to count schedules for repo %d: %v", resources.Repos[0].GetID(), err) } if int(count) != len(resources.Schedules) { t.Errorf("ListSchedulesForRepo() is %v, want %v", count, len(resources.Schedules)) } - if !cmp.Equal(list, []*library.Schedule{resources.Schedules[1], resources.Schedules[0]}, CmpOptApproxUpdatedAt()) { - t.Errorf("ListSchedulesForRepo() is %v, want %v", list, []*library.Schedule{resources.Schedules[1], resources.Schedules[0]}) + if !cmp.Equal(list, []*api.Schedule{resources.Schedules[1], resources.Schedules[0]}, CmpOptApproxUpdatedAt()) { + t.Errorf("ListSchedulesForRepo() is %v, want %v", list, []*api.Schedule{resources.Schedules[1], resources.Schedules[0]}) } methods["ListSchedulesForRepo"] = true // lookup the schedules by name for _, schedule := range resources.Schedules { - repo := resources.Repos[schedule.GetRepoID()-1] - got, err := db.GetScheduleForRepo(ctx, repo, schedule.GetName()) + repo := resources.Repos[schedule.GetRepo().GetID()-1] + got, err := db.GetScheduleForRepo(context.TODO(), repo, schedule.GetName()) if err != nil { t.Errorf("unable to get schedule %d for repo %d: %v", schedule.GetID(), repo.GetID(), err) } @@ -1358,7 +1372,7 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { // update the schedules for _, schedule := range resources.Schedules { schedule.SetUpdatedAt(time.Now().UTC().Unix()) - got, err := db.UpdateSchedule(ctx, schedule, true) + got, err := db.UpdateSchedule(context.TODO(), schedule, true) if err != nil { t.Errorf("unable to update schedule %d: %v", schedule.GetID(), err) } @@ -1372,13 +1386,29 @@ func testSchedules(t *testing.T, db Interface, resources *Resources) { // delete the schedules for _, schedule := range resources.Schedules { - err = db.DeleteSchedule(ctx, schedule) + err = db.DeleteSchedule(context.TODO(), schedule) if err != nil { t.Errorf("unable to delete schedule %d: %v", schedule.GetID(), err) } } methods["DeleteSchedule"] = true + // delete the repos + for _, repo := range resources.Repos { + err = db.DeleteRepo(context.TODO(), repo) + if err != nil { + t.Errorf("unable to delete repo %d: %v", repo.GetID(), err) + } + } + + // delete the owners + for _, user := range resources.Users { + err := db.DeleteUser(context.TODO(), user) + if err != nil { + t.Errorf("unable to delete user %d: %v", user.GetID(), err) + } + } + // ensure we called all the methods we expected to for method, called := range methods { if !called { @@ -2462,9 +2492,9 @@ func newResources() *Resources { pipelineTwo.SetTemplates(false) pipelineTwo.SetData([]byte("version: 1")) - scheduleOne := new(library.Schedule) + scheduleOne := new(api.Schedule) scheduleOne.SetID(1) - scheduleOne.SetRepoID(1) + scheduleOne.SetRepo(repoOne) scheduleOne.SetActive(true) scheduleOne.SetName("nightly") scheduleOne.SetEntry("0 0 * * *") @@ -2474,10 +2504,11 @@ func newResources() *Resources { scheduleOne.SetUpdatedBy("octokitty") scheduleOne.SetScheduledAt(time.Now().Add(time.Hour * 2).UTC().Unix()) scheduleOne.SetBranch("main") + scheduleOne.SetError("no version: YAML property provided") - scheduleTwo := new(library.Schedule) + scheduleTwo := new(api.Schedule) scheduleTwo.SetID(2) - scheduleTwo.SetRepoID(1) + scheduleTwo.SetRepo(repoOne) scheduleTwo.SetActive(true) scheduleTwo.SetName("hourly") scheduleTwo.SetEntry("0 * * * *") @@ -2487,6 +2518,7 @@ func newResources() *Resources { scheduleTwo.SetUpdatedBy("octokitty") scheduleTwo.SetScheduledAt(time.Now().Add(time.Hour * 2).UTC().Unix()) scheduleTwo.SetBranch("main") + scheduleTwo.SetError("no version: YAML property provided") secretOrg := new(library.Secret) secretOrg.SetID(1) @@ -2654,7 +2686,7 @@ func newResources() *Resources { Logs: []*library.Log{logServiceOne, logServiceTwo, logStepOne, logStepTwo}, Pipelines: []*library.Pipeline{pipelineOne, pipelineTwo}, Repos: []*api.Repo{repoOne, repoTwo}, - Schedules: []*library.Schedule{scheduleOne, scheduleTwo}, + Schedules: []*api.Schedule{scheduleOne, scheduleTwo}, Secrets: []*library.Secret{secretOrg, secretRepo, secretShared}, Services: []*library.Service{serviceOne, serviceTwo}, Steps: []*library.Step{stepOne, stepTwo}, diff --git a/database/resource.go b/database/resource.go index cbdad190c..21b805227 100644 --- a/database/resource.go +++ b/database/resource.go @@ -122,6 +122,7 @@ func (e *engine) NewResources(ctx context.Context) error { e.ScheduleInterface, err = schedule.New( schedule.WithContext(e.ctx), schedule.WithClient(e.client), + schedule.WithEncryptionKey(e.config.EncryptionKey), schedule.WithLogger(e.logger), schedule.WithSkipCreation(e.config.SkipCreation), ) diff --git a/database/schedule/count_active_test.go b/database/schedule/count_active_test.go index 6fb88849f..6c496965f 100644 --- a/database/schedule/count_active_test.go +++ b/database/schedule/count_active_test.go @@ -4,36 +4,78 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_CountActiveSchedules(t *testing.T) { - _scheduleOne := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -95,8 +137,8 @@ func TestSchedule_Engine_CountActiveSchedules(t *testing.T) { t.Errorf("CountActiveSchedules for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("CountActiveSchedules for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("CountActiveSchedules for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/count_repo_test.go b/database/schedule/count_repo_test.go index 9618f9cf8..360894fc1 100644 --- a/database/schedule/count_repo_test.go +++ b/database/schedule/count_repo_test.go @@ -4,46 +4,84 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { - _repo := testRepo() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() _repo.SetID(1) - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - - _scheduleOne := testSchedule() + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) + _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) + _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // create expected result in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(1) + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) // ensure the mock expects the query _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE repo_id = $1`).WithArgs(1).WillReturnRows(_rows) @@ -72,13 +110,13 @@ func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { failure: false, name: "postgres", database: _postgres, - want: 1, + want: 2, }, { failure: false, name: "sqlite3", database: _sqlite, - want: 1, + want: 2, }, } @@ -99,8 +137,8 @@ func TestSchedule_Engine_CountSchedulesForRepo(t *testing.T) { t.Errorf("CountSchedulesForRepo for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("CountSchedulesForRepo for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("CountSchedulesForRepo for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/count_test.go b/database/schedule/count_test.go index 98090ab8f..0338ff7fb 100644 --- a/database/schedule/count_test.go +++ b/database/schedule/count_test.go @@ -4,34 +4,78 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_CountSchedules(t *testing.T) { - _scheduleOne := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) + _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) + _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -93,8 +137,8 @@ func TestSchedule_Engine_CountSchedules(t *testing.T) { t.Errorf("CountSchedules for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("CountSchedules for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("CountSchedules for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/create.go b/database/schedule/create.go index bb7ac2beb..9b349a7d1 100644 --- a/database/schedule/create.go +++ b/database/schedule/create.go @@ -7,19 +7,19 @@ import ( "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // CreateSchedule creates a new schedule in the database. -func (e *engine) CreateSchedule(ctx context.Context, s *library.Schedule) (*library.Schedule, error) { +func (e *engine) CreateSchedule(ctx context.Context, s *api.Schedule) (*api.Schedule, error) { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("creating schedule %s in the database", s.GetName()) - // cast the library type to database type - schedule := database.ScheduleFromLibrary(s) + // cast the API type to database type + schedule := types.ScheduleFromAPI(s) // validate the necessary fields are populated err := schedule.Validate() @@ -28,7 +28,14 @@ func (e *engine) CreateSchedule(ctx context.Context, s *library.Schedule) (*libr } // send query to the database - result := e.client.Table(constants.TableSchedule).Create(schedule) + err = e.client.Table(constants.TableSchedule).Create(schedule).Error + if err != nil { + return nil, err + } + + // set repo to provided repo if creation successful + result := schedule.ToAPI() + result.SetRepo(s.GetRepo()) - return schedule.ToLibrary(), result.Error + return result, nil } diff --git a/database/schedule/create_test.go b/database/schedule/create_test.go index 7a2a42cc2..7858d4014 100644 --- a/database/schedule/create_test.go +++ b/database/schedule/create_test.go @@ -4,23 +4,64 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_CreateSchedule(t *testing.T) { - _schedule := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -30,9 +71,9 @@ func TestSchedule_Engine_CreateSchedule(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "schedules" -("repo_id","active","name","entry","created_at","created_by","updated_at","updated_by","scheduled_at","branch","id") -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`). - WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main", 1). +("repo_id","active","name","entry","created_at","created_by","updated_at","updated_by","scheduled_at","branch","error","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). + WithArgs(1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided", 1). WillReturnRows(_rows) _sqlite := testSqlite(t) @@ -73,8 +114,8 @@ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`). t.Errorf("CreateSchedule for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, _schedule) { - t.Errorf("CreateSchedule for %s returned %s, want %s", test.name, got, _schedule) + if diff := cmp.Diff(_schedule, got); diff != "" { + t.Errorf("CreateSchedule for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/delete.go b/database/schedule/delete.go index 0d3386da1..bc63c2f10 100644 --- a/database/schedule/delete.go +++ b/database/schedule/delete.go @@ -7,19 +7,19 @@ import ( "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // DeleteSchedule deletes an existing schedule from the database. -func (e *engine) DeleteSchedule(ctx context.Context, s *library.Schedule) error { +func (e *engine) DeleteSchedule(ctx context.Context, s *api.Schedule) error { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("deleting schedule %s in the database", s.GetName()) - // cast the library type to database type - schedule := database.ScheduleFromLibrary(s) + // cast the API type to database type + schedule := types.ScheduleFromAPI(s) // send query to the database return e.client. diff --git a/database/schedule/delete_test.go b/database/schedule/delete_test.go index 7a0a8ae76..d633bf38e 100644 --- a/database/schedule/delete_test.go +++ b/database/schedule/delete_test.go @@ -7,19 +7,60 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_DeleteSchedule(t *testing.T) { - _schedule := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() diff --git a/database/schedule/get.go b/database/schedule/get.go index 518bbd339..e8097bc51 100644 --- a/database/schedule/get.go +++ b/database/schedule/get.go @@ -5,21 +5,23 @@ package schedule import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // GetSchedule gets a schedule by ID from the database. -func (e *engine) GetSchedule(ctx context.Context, id int64) (*library.Schedule, error) { +func (e *engine) GetSchedule(ctx context.Context, id int64) (*api.Schedule, error) { e.logger.Tracef("getting schedule %d from the database", id) // variable to store query results - s := new(database.Schedule) + s := new(types.Schedule) // send query to the database and store result in variable err := e.client. Table(constants.TableSchedule). + Preload("Repo"). + Preload("Repo.Owner"). Where("id = ?", id). Take(s). Error @@ -27,5 +29,11 @@ func (e *engine) GetSchedule(ctx context.Context, id int64) (*library.Schedule, return nil, err } - return s.ToLibrary(), nil + // decrypt hash value for repo + err = s.Repo.Decrypt(e.config.EncryptionKey) + if err != nil { + e.logger.Errorf("unable to decrypt repo %d: %v", s.Repo.ID.Int64, err) + } + + return s.ToAPI(), nil } diff --git a/database/schedule/get_repo.go b/database/schedule/get_repo.go index 6bc362e3d..0af4d1f7d 100644 --- a/database/schedule/get_repo.go +++ b/database/schedule/get_repo.go @@ -8,13 +8,12 @@ import ( "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // GetScheduleForRepo gets a schedule by repo ID and name from the database. -func (e *engine) GetScheduleForRepo(ctx context.Context, r *api.Repo, name string) (*library.Schedule, error) { +func (e *engine) GetScheduleForRepo(ctx context.Context, r *api.Repo, name string) (*api.Schedule, error) { e.logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -22,11 +21,13 @@ func (e *engine) GetScheduleForRepo(ctx context.Context, r *api.Repo, name strin }).Tracef("getting schedule %s/%s from the database", r.GetFullName(), name) // variable to store query results - s := new(database.Schedule) + s := new(types.Schedule) // send query to the database and store result in variable err := e.client. Table(constants.TableSchedule). + Preload("Repo"). + Preload("Repo.Owner"). Where("repo_id = ?", r.GetID()). Where("name = ?", name). Take(s). @@ -35,5 +36,11 @@ func (e *engine) GetScheduleForRepo(ctx context.Context, r *api.Repo, name strin return nil, err } - return s.ToLibrary(), nil + // decrypt hash value for repo + err = s.Repo.Decrypt(e.config.EncryptionKey) + if err != nil { + e.logger.Errorf("unable to decrypt repo %d: %v", s.Repo.ID.Int64, err) + } + + return s.ToAPI(), nil } diff --git a/database/schedule/get_repo_test.go b/database/schedule/get_repo_test.go index 0d6a6184c..40a92a761 100644 --- a/database/schedule/get_repo_test.go +++ b/database/schedule/get_repo_test.go @@ -4,42 +4,86 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { - _repo := testRepo() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() _repo.SetID(1) - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - - _schedule := testSchedule() + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // create expected result in mock _rows := sqlmock.NewRows( - []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch"}, - ).AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main") + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch", "error"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided") + + _repoRows := sqlmock.NewRows( + []string{"id", "user_id", "hash", "org", "name", "full_name", "link", "clone", "branch", "topics", "build_limit", "timeout", "counter", "visibility", "private", "trusted", "active", "allow_events", "pipeline_type", "previous_name", "approve_build"}). + AddRow(1, 1, "MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy", "github", "octocat", "github/octocat", "https://github.com/github/octocat", "https://github.com/github/octocat.git", "main", "{cloud,security}", 10, 30, 0, "public", false, false, true, 1, "", "", constants.ApproveNever) + + _userRows := sqlmock.NewRows( + []string{"id", "name", "token", "refresh_token", "favorites", "active", "admin", "dashboards"}). + AddRow(1, "octocat", "superSecretToken", "superSecretRefreshToken", "{}", true, false, "{}") // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE repo_id = $1 AND name = $2 LIMIT $3`).WithArgs(1, "nightly", 1).WillReturnRows(_rows) + _mock.ExpectQuery(`SELECT * FROM "repos" WHERE "repos"."id" = $1`).WithArgs(1).WillReturnRows(_repoRows) + _mock.ExpectQuery(`SELECT * FROM "users" WHERE "users"."id" = $1`).WithArgs(1).WillReturnRows(_userRows) _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() @@ -49,12 +93,32 @@ func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { t.Errorf("unable to create test schedule for sqlite: %v", err) } + err = _sqlite.client.AutoMigrate(&types.Repo{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(types.RepoFromAPI(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&types.User{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableUser).Create(types.UserFromAPI(_owner)).Error + if err != nil { + t.Errorf("unable to create test user for sqlite: %v", err) + } + // setup tests tests := []struct { failure bool name string database *engine - want *library.Schedule + want *api.Schedule }{ { failure: false, @@ -87,8 +151,8 @@ func TestSchedule_Engine_GetScheduleForRepo(t *testing.T) { t.Errorf("GetScheduleForRepo for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetScheduleForRepo for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("GetScheduleForRepo for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/get_test.go b/database/schedule/get_test.go index 3c62b8908..2f5b81952 100644 --- a/database/schedule/get_test.go +++ b/database/schedule/get_test.go @@ -4,36 +4,86 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_GetSchedule(t *testing.T) { - _schedule := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // create expected result in mock _rows := sqlmock.NewRows( - []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch"}, - ).AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main") + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch", "error"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided") + + _repoRows := sqlmock.NewRows( + []string{"id", "user_id", "hash", "org", "name", "full_name", "link", "clone", "branch", "topics", "build_limit", "timeout", "counter", "visibility", "private", "trusted", "active", "allow_events", "pipeline_type", "previous_name", "approve_build"}). + AddRow(1, 1, "MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy", "github", "octocat", "github/octocat", "https://github.com/github/octocat", "https://github.com/github/octocat.git", "main", "{cloud,security}", 10, 30, 0, "public", false, false, true, 1, "", "", constants.ApproveNever) + + _userRows := sqlmock.NewRows( + []string{"id", "name", "token", "refresh_token", "favorites", "active", "admin", "dashboards"}). + AddRow(1, "octocat", "superSecretToken", "superSecretRefreshToken", "{}", true, false, "{}") // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE id = $1 LIMIT $2`).WithArgs(1, 1).WillReturnRows(_rows) + _mock.ExpectQuery(`SELECT * FROM "repos" WHERE "repos"."id" = $1`).WithArgs(1).WillReturnRows(_repoRows) + _mock.ExpectQuery(`SELECT * FROM "users" WHERE "users"."id" = $1`).WithArgs(1).WillReturnRows(_userRows) _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() @@ -43,12 +93,32 @@ func TestSchedule_Engine_GetSchedule(t *testing.T) { t.Errorf("unable to create test schedule for sqlite: %v", err) } + err = _sqlite.client.AutoMigrate(&types.Repo{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(types.RepoFromAPI(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&types.User{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableUser).Create(types.UserFromAPI(_owner)).Error + if err != nil { + t.Errorf("unable to create test user for sqlite: %v", err) + } + // setup tests tests := []struct { failure bool name string database *engine - want *library.Schedule + want *api.Schedule }{ { failure: false, @@ -81,8 +151,8 @@ func TestSchedule_Engine_GetSchedule(t *testing.T) { t.Errorf("GetSchedule for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetSchedule for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("GetSchedule for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/interface.go b/database/schedule/interface.go index b25b3c363..f4d284b45 100644 --- a/database/schedule/interface.go +++ b/database/schedule/interface.go @@ -6,7 +6,6 @@ import ( "context" api "github.com/go-vela/server/api/types" - "github.com/go-vela/types/library" ) // ScheduleInterface represents the Vela interface for schedule @@ -32,19 +31,19 @@ type ScheduleInterface interface { // CountSchedulesForRepo defines a function that gets the count of schedules by repo ID. CountSchedulesForRepo(context.Context, *api.Repo) (int64, error) // CreateSchedule defines a function that creates a new schedule. - CreateSchedule(context.Context, *library.Schedule) (*library.Schedule, error) + CreateSchedule(context.Context, *api.Schedule) (*api.Schedule, error) // DeleteSchedule defines a function that deletes an existing schedule. - DeleteSchedule(context.Context, *library.Schedule) error + DeleteSchedule(context.Context, *api.Schedule) error // GetSchedule defines a function that gets a schedule by ID. - GetSchedule(context.Context, int64) (*library.Schedule, error) + GetSchedule(context.Context, int64) (*api.Schedule, error) // GetScheduleForRepo defines a function that gets a schedule by repo ID and name. - GetScheduleForRepo(context.Context, *api.Repo, string) (*library.Schedule, error) + GetScheduleForRepo(context.Context, *api.Repo, string) (*api.Schedule, error) // ListActiveSchedules defines a function that gets a list of all active schedules. - ListActiveSchedules(context.Context) ([]*library.Schedule, error) + ListActiveSchedules(context.Context) ([]*api.Schedule, error) // ListSchedules defines a function that gets a list of all schedules. - ListSchedules(context.Context) ([]*library.Schedule, error) + ListSchedules(context.Context) ([]*api.Schedule, error) // ListSchedulesForRepo defines a function that gets a list of schedules by repo ID. - ListSchedulesForRepo(context.Context, *api.Repo, int, int) ([]*library.Schedule, int64, error) + ListSchedulesForRepo(context.Context, *api.Repo, int, int) ([]*api.Schedule, int64, error) // UpdateSchedule defines a function that updates an existing schedule. - UpdateSchedule(context.Context, *library.Schedule, bool) (*library.Schedule, error) + UpdateSchedule(context.Context, *api.Schedule, bool) (*api.Schedule, error) } diff --git a/database/schedule/list.go b/database/schedule/list.go index f3196651f..7df0c2248 100644 --- a/database/schedule/list.go +++ b/database/schedule/list.go @@ -5,19 +5,19 @@ package schedule import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListSchedules gets a list of all schedules from the database. -func (e *engine) ListSchedules(ctx context.Context) ([]*library.Schedule, error) { +func (e *engine) ListSchedules(ctx context.Context) ([]*api.Schedule, error) { e.logger.Trace("listing all schedules from the database") // variables to store query results and return value count := int64(0) - s := new([]database.Schedule) - schedules := []*library.Schedule{} + s := new([]types.Schedule) + schedules := []*api.Schedule{} // count the results count, err := e.CountSchedules(ctx) @@ -33,6 +33,8 @@ func (e *engine) ListSchedules(ctx context.Context) ([]*library.Schedule, error) // send query to the database and store result in variable err = e.client. Table(constants.TableSchedule). + Preload("Repo"). + Preload("Repo.Owner"). Find(&s). Error if err != nil { @@ -44,8 +46,14 @@ func (e *engine) ListSchedules(ctx context.Context) ([]*library.Schedule, error) // https://golang.org/doc/faq#closures_and_goroutines tmp := schedule + // decrypt hash value for repo + err = tmp.Repo.Decrypt(e.config.EncryptionKey) + if err != nil { + e.logger.Errorf("unable to decrypt repo %d: %v", tmp.ID.Int64, err) + } + // convert query result to API type - schedules = append(schedules, tmp.ToLibrary()) + schedules = append(schedules, tmp.ToAPI()) } return schedules, nil diff --git a/database/schedule/list_active.go b/database/schedule/list_active.go index b91ad63e8..d53b8201d 100644 --- a/database/schedule/list_active.go +++ b/database/schedule/list_active.go @@ -5,19 +5,19 @@ package schedule import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListActiveSchedules gets a list of all active schedules from the database. -func (e *engine) ListActiveSchedules(ctx context.Context) ([]*library.Schedule, error) { +func (e *engine) ListActiveSchedules(ctx context.Context) ([]*api.Schedule, error) { e.logger.Trace("listing all active schedules from the database") // variables to store query results and return value count := int64(0) - s := new([]database.Schedule) - schedules := []*library.Schedule{} + s := new([]types.Schedule) + schedules := []*api.Schedule{} // count the results count, err := e.CountActiveSchedules(ctx) @@ -33,6 +33,8 @@ func (e *engine) ListActiveSchedules(ctx context.Context) ([]*library.Schedule, // send query to the database and store result in variable err = e.client. Table(constants.TableSchedule). + Preload("Repo"). + Preload("Repo.Owner"). Where("active = ?", true). Find(&s). Error @@ -45,8 +47,14 @@ func (e *engine) ListActiveSchedules(ctx context.Context) ([]*library.Schedule, // https://golang.org/doc/faq#closures_and_goroutines tmp := schedule + // decrypt hash value for repo + err = tmp.Repo.Decrypt(e.config.EncryptionKey) + if err != nil { + e.logger.Errorf("unable to decrypt repo %d: %v", tmp.Repo.ID.Int64, err) + } + // convert query result to API type - schedules = append(schedules, tmp.ToLibrary()) + schedules = append(schedules, tmp.ToAPI()) } return schedules, nil diff --git a/database/schedule/list_active_test.go b/database/schedule/list_active_test.go index 8154780bd..e3179e020 100644 --- a/database/schedule/list_active_test.go +++ b/database/schedule/list_active_test.go @@ -4,55 +4,106 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { - _scheduleOne := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // create expected result in mock - _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + _rows := sqlmock.NewRows([]string{"count"}).AddRow(1) // ensure the mock expects the query _mock.ExpectQuery(`SELECT count(*) FROM "schedules" WHERE active = $1`).WithArgs(true).WillReturnRows(_rows) // create expected result in mock _rows = sqlmock.NewRows( - []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch"}). - AddRow(1, 1, true, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main") + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch", "error"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided") + + _repoRows := sqlmock.NewRows( + []string{"id", "user_id", "hash", "org", "name", "full_name", "link", "clone", "branch", "topics", "build_limit", "timeout", "counter", "visibility", "private", "trusted", "active", "allow_events", "pipeline_type", "previous_name", "approve_build"}). + AddRow(1, 1, "MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy", "github", "octocat", "github/octocat", "https://github.com/github/octocat", "https://github.com/github/octocat.git", "main", "{cloud,security}", 10, 30, 0, "public", false, false, true, 1, "", "", constants.ApproveNever) + + _userRows := sqlmock.NewRows( + []string{"id", "name", "token", "refresh_token", "favorites", "active", "admin", "dashboards"}). + AddRow(1, "octocat", "superSecretToken", "superSecretRefreshToken", "{}", true, false, "{}") // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE active = $1`).WithArgs(true).WillReturnRows(_rows) + _mock.ExpectQuery(`SELECT * FROM "repos" WHERE "repos"."id" = $1`).WithArgs(1).WillReturnRows(_repoRows) + _mock.ExpectQuery(`SELECT * FROM "users" WHERE "users"."id" = $1`).WithArgs(1).WillReturnRows(_userRows) _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() @@ -66,25 +117,44 @@ func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { if err != nil { t.Errorf("unable to create test schedule for sqlite: %v", err) } + err = _sqlite.client.AutoMigrate(&types.Repo{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(types.RepoFromAPI(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&types.User{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableUser).Create(types.UserFromAPI(_owner)).Error + if err != nil { + t.Errorf("unable to create test user for sqlite: %v", err) + } // setup tests tests := []struct { failure bool name string database *engine - want []*library.Schedule + want []*api.Schedule }{ { failure: false, name: "postgres", database: _postgres, - want: []*library.Schedule{_scheduleOne}, + want: []*api.Schedule{_scheduleOne}, }, { failure: false, name: "sqlite3", database: _sqlite, - want: []*library.Schedule{_scheduleOne}, + want: []*api.Schedule{_scheduleOne}, }, } @@ -105,8 +175,8 @@ func TestSchedule_Engine_ListActiveSchedules(t *testing.T) { t.Errorf("ListActiveSchedules for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("ListActiveSchedules for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("ListActiveSchedules for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/list_repo.go b/database/schedule/list_repo.go index 9f3de5e5a..d26f6a198 100644 --- a/database/schedule/list_repo.go +++ b/database/schedule/list_repo.go @@ -8,13 +8,12 @@ import ( "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListSchedulesForRepo gets a list of schedules by repo ID from the database. -func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, perPage int) ([]*library.Schedule, int64, error) { +func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, perPage int) ([]*api.Schedule, int64, error) { e.logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -22,8 +21,8 @@ func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, pe // variables to store query results and return value count := int64(0) - s := new([]database.Schedule) - schedules := []*library.Schedule{} + s := new([]types.Schedule) + schedules := []*api.Schedule{} // count the results count, err := e.CountSchedulesForRepo(ctx, r) @@ -42,6 +41,8 @@ func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, pe // send query to the database and store result in variable err = e.client. Table(constants.TableSchedule). + Preload("Repo"). + Preload("Repo.Owner"). Where("repo_id = ?", r.GetID()). Order("id DESC"). Limit(perPage). @@ -57,8 +58,14 @@ func (e *engine) ListSchedulesForRepo(ctx context.Context, r *api.Repo, page, pe // https://golang.org/doc/faq#closures_and_goroutines tmp := schedule - // convert query result to library type - schedules = append(schedules, tmp.ToLibrary()) + // decrypt hash value for repo + err = tmp.Repo.Decrypt(e.config.EncryptionKey) + if err != nil { + e.logger.Errorf("unable to decrypt repo %d: %v", tmp.Repo.ID.Int64, err) + } + + // convert query result to API type + schedules = append(schedules, tmp.ToAPI()) } return schedules, count, nil diff --git a/database/schedule/list_repo_test.go b/database/schedule/list_repo_test.go index 4753a2b34..43b1fcf58 100644 --- a/database/schedule/list_repo_test.go +++ b/database/schedule/list_repo_test.go @@ -4,42 +4,79 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { - _repo := testRepo() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() _repo.SetID(1) - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - - _scheduleOne := testSchedule() + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) + _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) + _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -52,11 +89,22 @@ func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { // create expected result in mock _rows = sqlmock.NewRows( - []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch"}). - AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main") + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch", "error"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided"). + AddRow(2, 1, false, "hourly", "0 * * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided") + + _repoRows := sqlmock.NewRows( + []string{"id", "user_id", "hash", "org", "name", "full_name", "link", "clone", "branch", "topics", "build_limit", "timeout", "counter", "visibility", "private", "trusted", "active", "allow_events", "pipeline_type", "previous_name", "approve_build"}). + AddRow(1, 1, "MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy", "github", "octocat", "github/octocat", "https://github.com/github/octocat", "https://github.com/github/octocat.git", "main", "{cloud,security}", 10, 30, 0, "public", false, false, true, 1, "", "", constants.ApproveNever) + + _userRows := sqlmock.NewRows( + []string{"id", "name", "token", "refresh_token", "favorites", "active", "admin", "dashboards"}). + AddRow(1, "octocat", "superSecretToken", "superSecretRefreshToken", "{}", true, false, "{}") // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "schedules" WHERE repo_id = $1 ORDER BY id DESC LIMIT $2`).WithArgs(1, 10).WillReturnRows(_rows) + _mock.ExpectQuery(`SELECT * FROM "repos" WHERE "repos"."id" = $1`).WithArgs(1).WillReturnRows(_repoRows) + _mock.ExpectQuery(`SELECT * FROM "users" WHERE "users"."id" = $1`).WithArgs(1).WillReturnRows(_userRows) _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() @@ -71,24 +119,44 @@ func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { t.Errorf("unable to create test schedule for sqlite: %v", err) } + err = _sqlite.client.AutoMigrate(&types.Repo{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(types.RepoFromAPI(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&types.User{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableUser).Create(types.UserFromAPI(_owner)).Error + if err != nil { + t.Errorf("unable to create test user for sqlite: %v", err) + } + // setup tests tests := []struct { failure bool name string database *engine - want []*library.Schedule + want []*api.Schedule }{ { failure: false, name: "postgres", database: _postgres, - want: []*library.Schedule{_scheduleOne}, + want: []*api.Schedule{_scheduleOne, _scheduleTwo}, }, { failure: false, name: "sqlite3", database: _sqlite, - want: []*library.Schedule{_scheduleOne}, + want: []*api.Schedule{_scheduleTwo, _scheduleOne}, }, } @@ -109,8 +177,8 @@ func TestSchedule_Engine_ListSchedulesForRepo(t *testing.T) { t.Errorf("ListSchedulesForRepo for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("ListSchedulesForRepo for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("ListSchedulesForRepo for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/list_test.go b/database/schedule/list_test.go index d8a5b2ae8..8338c5dc1 100644 --- a/database/schedule/list_test.go +++ b/database/schedule/list_test.go @@ -4,36 +4,79 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/server/database/types" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_ListSchedules(t *testing.T) { - _scheduleOne := testSchedule() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() + _repo.SetID(1) + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _scheduleOne := testutils.APISchedule() _scheduleOne.SetID(1) - _scheduleOne.SetRepoID(1) + _scheduleOne.SetRepo(_repo) + _scheduleOne.SetActive(true) _scheduleOne.SetName("nightly") _scheduleOne.SetEntry("0 0 * * *") - _scheduleOne.SetCreatedAt(1) - _scheduleOne.SetCreatedBy("user1") - _scheduleOne.SetUpdatedAt(1) - _scheduleOne.SetUpdatedBy("user2") + _scheduleOne.SetCreatedAt(1713476291) + _scheduleOne.SetCreatedBy("octocat") + _scheduleOne.SetUpdatedAt(3013476291) + _scheduleOne.SetUpdatedBy("octokitty") + _scheduleOne.SetScheduledAt(2013476291) _scheduleOne.SetBranch("main") + _scheduleOne.SetError("no version: YAML property provided") - _scheduleTwo := testSchedule() + _scheduleTwo := testutils.APISchedule() _scheduleTwo.SetID(2) - _scheduleTwo.SetRepoID(2) + _scheduleTwo.SetRepo(_repo) + _scheduleTwo.SetActive(false) _scheduleTwo.SetName("hourly") _scheduleTwo.SetEntry("0 * * * *") - _scheduleTwo.SetCreatedAt(1) - _scheduleTwo.SetCreatedBy("user1") - _scheduleTwo.SetUpdatedAt(1) - _scheduleTwo.SetUpdatedBy("user2") + _scheduleTwo.SetCreatedAt(1713476291) + _scheduleTwo.SetCreatedBy("octocat") + _scheduleTwo.SetUpdatedAt(3013476291) + _scheduleTwo.SetUpdatedBy("octokitty") + _scheduleTwo.SetScheduledAt(2013476291) _scheduleTwo.SetBranch("main") + _scheduleTwo.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -46,12 +89,22 @@ func TestSchedule_Engine_ListSchedules(t *testing.T) { // create expected result in mock _rows = sqlmock.NewRows( - []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch"}). - AddRow(1, 1, false, "nightly", "0 0 * * *", 1, "user1", 1, "user2", nil, "main"). - AddRow(2, 2, false, "hourly", "0 * * * *", 1, "user1", 1, "user2", nil, "main") + []string{"id", "repo_id", "active", "name", "entry", "created_at", "created_by", "updated_at", "updated_by", "scheduled_at", "branch", "error"}). + AddRow(1, 1, true, "nightly", "0 0 * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided"). + AddRow(2, 1, false, "hourly", "0 * * * *", 1713476291, "octocat", 3013476291, "octokitty", 2013476291, "main", "no version: YAML property provided") + + _repoRows := sqlmock.NewRows( + []string{"id", "user_id", "hash", "org", "name", "full_name", "link", "clone", "branch", "topics", "build_limit", "timeout", "counter", "visibility", "private", "trusted", "active", "allow_events", "pipeline_type", "previous_name", "approve_build"}). + AddRow(1, 1, "MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy", "github", "octocat", "github/octocat", "https://github.com/github/octocat", "https://github.com/github/octocat.git", "main", "{cloud,security}", 10, 30, 0, "public", false, false, true, 1, "", "", constants.ApproveNever) + + _userRows := sqlmock.NewRows( + []string{"id", "name", "token", "refresh_token", "favorites", "active", "admin", "dashboards"}). + AddRow(1, "octocat", "superSecretToken", "superSecretRefreshToken", "{}", true, false, "{}") // ensure the mock expects the query _mock.ExpectQuery(`SELECT * FROM "schedules"`).WillReturnRows(_rows) + _mock.ExpectQuery(`SELECT * FROM "repos" WHERE "repos"."id" = $1`).WithArgs(1).WillReturnRows(_repoRows) + _mock.ExpectQuery(`SELECT * FROM "users" WHERE "users"."id" = $1`).WithArgs(1).WillReturnRows(_userRows) _sqlite := testSqlite(t) defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() @@ -66,24 +119,44 @@ func TestSchedule_Engine_ListSchedules(t *testing.T) { t.Errorf("unable to create test schedule for sqlite: %v", err) } + err = _sqlite.client.AutoMigrate(&types.Repo{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableRepo).Create(types.RepoFromAPI(_repo)).Error + if err != nil { + t.Errorf("unable to create test repo for sqlite: %v", err) + } + + err = _sqlite.client.AutoMigrate(&types.User{}) + if err != nil { + t.Errorf("unable to create build table for sqlite: %v", err) + } + + err = _sqlite.client.Table(constants.TableUser).Create(types.UserFromAPI(_owner)).Error + if err != nil { + t.Errorf("unable to create test user for sqlite: %v", err) + } + // setup tests tests := []struct { failure bool name string database *engine - want []*library.Schedule + want []*api.Schedule }{ { failure: false, name: "postgres", database: _postgres, - want: []*library.Schedule{_scheduleOne, _scheduleTwo}, + want: []*api.Schedule{_scheduleOne, _scheduleTwo}, }, { failure: false, name: "sqlite3", database: _sqlite, - want: []*library.Schedule{_scheduleOne, _scheduleTwo}, + want: []*api.Schedule{_scheduleOne, _scheduleTwo}, }, } @@ -104,8 +177,8 @@ func TestSchedule_Engine_ListSchedules(t *testing.T) { t.Errorf("ListSchedules for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, test.want) { - t.Errorf("ListSchedules for %s is %v, want %v", test.name, got, test.want) + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("ListSchedules for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/schedule/opts.go b/database/schedule/opts.go index 7eb9c2adb..c3a0af1eb 100644 --- a/database/schedule/opts.go +++ b/database/schedule/opts.go @@ -22,6 +22,16 @@ func WithClient(client *gorm.DB) EngineOpt { } } +// WithEncryptionKey sets the encryption key in the database engine for Schedules. +func WithEncryptionKey(key string) EngineOpt { + return func(e *engine) error { + // set the encryption key in the schedule engine + e.config.EncryptionKey = key + + return nil + } +} + // WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Schedules. func WithLogger(logger *logrus.Entry) EngineOpt { return func(e *engine) error { diff --git a/database/schedule/schedule.go b/database/schedule/schedule.go index b338d2f95..b7a83b2dd 100644 --- a/database/schedule/schedule.go +++ b/database/schedule/schedule.go @@ -15,6 +15,8 @@ import ( type ( // config represents the settings required to create the engine that implements the ScheduleInterface interface. config struct { + // specifies the encryption key to use for the Schedule engine + EncryptionKey string // specifies to skip creating tables and indexes for the Schedule engine SkipCreation bool } diff --git a/database/schedule/schedule_test.go b/database/schedule/schedule_test.go index 5c227f4c2..a7d4cb936 100644 --- a/database/schedule/schedule_test.go +++ b/database/schedule/schedule_test.go @@ -14,9 +14,6 @@ import ( "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" - - api "github.com/go-vela/server/api/types" - "github.com/go-vela/types/library" ) func TestSchedule_New(t *testing.T) { @@ -173,46 +170,6 @@ func testSqlite(t *testing.T) *engine { return _engine } -// testSchedule is a test helper function to create an API Schedule type with all fields set to their zero values. -func testSchedule() *library.Schedule { - return &library.Schedule{ - ID: new(int64), - RepoID: new(int64), - Active: new(bool), - Name: new(string), - Entry: new(string), - CreatedAt: new(int64), - CreatedBy: new(string), - UpdatedAt: new(int64), - UpdatedBy: new(string), - ScheduledAt: new(int64), - Branch: new(string), - } -} - -// testRepo is a test helper function to create a library Repo type with all fields set to their zero values. -func testRepo() *api.Repo { - return &api.Repo{ - ID: new(int64), - BuildLimit: new(int64), - Timeout: new(int64), - Counter: new(int), - PipelineType: new(string), - Hash: new(string), - Org: new(string), - Name: new(string), - FullName: new(string), - Link: new(string), - Clone: new(string), - Branch: new(string), - Visibility: new(string), - PreviousName: new(string), - Private: new(bool), - Trusted: new(bool), - Active: new(bool), - } -} - // This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values // that are otherwise not easily compared. These typically would be values generated // before adding or updating them in the database. diff --git a/database/schedule/table.go b/database/schedule/table.go index ea8e99cb3..82ae8fdaf 100644 --- a/database/schedule/table.go +++ b/database/schedule/table.go @@ -25,6 +25,7 @@ schedules ( updated_by VARCHAR(250), scheduled_at INTEGER, branch VARCHAR(250), + error VARCHAR(250), UNIQUE(repo_id, name) ); ` @@ -45,6 +46,7 @@ schedules ( updated_by TEXT, scheduled_at INTEGER, branch TEXT, + error TEXT, UNIQUE(repo_id, name) ); ` diff --git a/database/schedule/update.go b/database/schedule/update.go index ca90e13b4..790ef89f3 100644 --- a/database/schedule/update.go +++ b/database/schedule/update.go @@ -7,19 +7,19 @@ import ( "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // UpdateSchedule updates an existing schedule in the database. -func (e *engine) UpdateSchedule(ctx context.Context, s *library.Schedule, fields bool) (*library.Schedule, error) { +func (e *engine) UpdateSchedule(ctx context.Context, s *api.Schedule, fields bool) (*api.Schedule, error) { e.logger.WithFields(logrus.Fields{ "schedule": s.GetName(), }).Tracef("updating schedule %s in the database", s.GetName()) - // cast the library type to database type - schedule := database.ScheduleFromLibrary(s) + // cast the API type to database type + schedule := types.ScheduleFromAPI(s) // validate the necessary fields are populated err := schedule.Validate() @@ -37,5 +37,13 @@ func (e *engine) UpdateSchedule(ctx context.Context, s *library.Schedule, fields err = e.client.Table(constants.TableSchedule).Model(schedule).UpdateColumn("scheduled_at", s.GetScheduledAt()).Error } - return schedule.ToLibrary(), err + if err != nil { + return nil, err + } + + // set repo to provided repo if creation successful + result := schedule.ToAPI() + result.SetRepo(s.GetRepo()) + + return result, nil } diff --git a/database/schedule/update_test.go b/database/schedule/update_test.go index 45f9dbdf8..f0b1ea2c0 100644 --- a/database/schedule/update_test.go +++ b/database/schedule/update_test.go @@ -4,38 +4,73 @@ package schedule import ( "context" - "reflect" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/google/go-cmp/cmp" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" ) func TestSchedule_Engine_UpdateSchedule_Config(t *testing.T) { - _repo := testRepo() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() _repo.SetID(1) - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - - _schedule := testSchedule() + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // ensure the mock expects the query _mock.ExpectExec(`UPDATE "schedules" -SET "repo_id"=$1,"active"=$2,"name"=$3,"entry"=$4,"created_at"=$5,"created_by"=$6,"updated_at"=$7,"updated_by"=$8,"scheduled_at"=$9,"branch"=$10 -WHERE "id" = $11`). - WithArgs(1, false, "nightly", "0 0 * * *", 1, "user1", NowTimestamp{}, "user2", nil, "main", 1). +SET "repo_id"=$1,"active"=$2,"name"=$3,"entry"=$4,"created_at"=$5,"created_by"=$6,"updated_at"=$7,"updated_by"=$8,"scheduled_at"=$9,"branch"=$10,"error"=$11 +WHERE "id" = $12`). + WithArgs(1, true, "nightly", "0 0 * * *", 1713476291, "octocat", NowTimestamp{}, "octokitty", 2013476291, "main", "no version: YAML property provided", 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) @@ -82,38 +117,68 @@ WHERE "id" = $11`). t.Errorf("UpdateSchedule for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, _schedule) { - t.Errorf("UpdateSchedule for %s returned %s, want %s", test.name, got, _schedule) + if diff := cmp.Diff(_schedule, got); diff != "" { + t.Errorf("UpdateSchedule for %s mismatch (-want +got):\n%s", test.name, diff) } }) } } func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { - _repo := testRepo() + // setup types + _owner := testutils.APIUser() + _owner.SetID(1) + _owner.SetName("octocat") + _owner.SetToken("superSecretToken") + _owner.SetRefreshToken("superSecretRefreshToken") + _owner.SetFavorites([]string{"github/octocat"}) + _owner.SetActive(true) + _owner.SetAdmin(false) + _owner.SetDashboards([]string{"45bcf19b-c151-4e2d-b8c6-80a62ba2eae7"}) + + _repo := testutils.APIRepo() _repo.SetID(1) - _repo.SetOrg("foo") - _repo.SetName("bar") - _repo.SetFullName("foo/bar") - - _schedule := testSchedule() + _repo.SetOwner(_owner.Crop()) + _repo.SetHash("MzM4N2MzMDAtNmY4Mi00OTA5LWFhZDAtNWIzMTlkNTJkODMy") + _repo.SetOrg("github") + _repo.SetName("octocat") + _repo.SetFullName("github/octocat") + _repo.SetLink("https://github.com/github/octocat") + _repo.SetClone("https://github.com/github/octocat.git") + _repo.SetBranch("main") + _repo.SetTopics([]string{"cloud", "security"}) + _repo.SetBuildLimit(10) + _repo.SetTimeout(30) + _repo.SetCounter(0) + _repo.SetVisibility("public") + _repo.SetPrivate(false) + _repo.SetTrusted(false) + _repo.SetActive(true) + _repo.SetAllowEvents(api.NewEventsFromMask(1)) + _repo.SetPipelineType("") + _repo.SetPreviousName("") + _repo.SetApproveBuild(constants.ApproveNever) + + _schedule := testutils.APISchedule() _schedule.SetID(1) - _schedule.SetRepoID(1) + _schedule.SetRepo(_repo) + _schedule.SetActive(true) _schedule.SetName("nightly") _schedule.SetEntry("0 0 * * *") - _schedule.SetCreatedAt(1) - _schedule.SetCreatedBy("user1") - _schedule.SetUpdatedAt(1) - _schedule.SetUpdatedBy("user2") - _schedule.SetScheduledAt(1) + _schedule.SetCreatedAt(1713476291) + _schedule.SetCreatedBy("octocat") + _schedule.SetUpdatedAt(3013476291) + _schedule.SetUpdatedBy("octokitty") + _schedule.SetScheduledAt(2013476291) _schedule.SetBranch("main") + _schedule.SetError("no version: YAML property provided") _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() // ensure the mock expects the query _mock.ExpectExec(`UPDATE "schedules" SET "scheduled_at"=$1 WHERE "id" = $2`). - WithArgs(1, 1). + WithArgs(2013476291, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) @@ -159,8 +224,8 @@ func TestSchedule_Engine_UpdateSchedule_NotConfig(t *testing.T) { t.Errorf("UpdateSchedule for %s returned err: %v", test.name, err) } - if !reflect.DeepEqual(got, _schedule) { - t.Errorf("CreateSchedule for %s returned %s, want %s", test.name, got, _schedule) + if diff := cmp.Diff(_schedule, got); diff != "" { + t.Errorf("UpdateSchedule for %s mismatch (-want +got):\n%s", test.name, diff) } }) } diff --git a/database/testutils/api_resources.go b/database/testutils/api_resources.go index 78aec3aa6..06b34eba8 100644 --- a/database/testutils/api_resources.go +++ b/database/testutils/api_resources.go @@ -169,6 +169,23 @@ func APILog() *library.Log { } } +func APISchedule() *api.Schedule { + return &api.Schedule{ + ID: new(int64), + Repo: APIRepo(), + Active: new(bool), + Name: new(string), + Entry: new(string), + CreatedAt: new(int64), + CreatedBy: new(string), + UpdatedAt: new(int64), + UpdatedBy: new(string), + ScheduledAt: new(int64), + Branch: new(string), + Error: new(string), + } +} + func APIService() *library.Service { return &library.Service{ ID: new(int64), diff --git a/database/types/repo_test.go b/database/types/repo_test.go index f060e9141..d41ca7f8b 100644 --- a/database/types/repo_test.go +++ b/database/types/repo_test.go @@ -353,7 +353,7 @@ func TestTypes_RepoFromAPI(t *testing.T) { got := RepoFromAPI(r) if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("FromAPI() mismatch (-want +got):\n%s", diff) + t.Errorf("RepoFromAPI() mismatch (-want +got):\n%s", diff) } } diff --git a/database/types/schedule.go b/database/types/schedule.go new file mode 100644 index 000000000..6d2ada21c --- /dev/null +++ b/database/types/schedule.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "errors" + + "github.com/adhocore/gronx" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/util" +) + +var ( + // ErrEmptyScheduleEntry defines the error type when a Schedule type has an empty Entry field provided. + ErrEmptyScheduleEntry = errors.New("empty schedule entry provided") + + // ErrEmptyScheduleName defines the error type when a Schedule type has an empty Name field provided. + ErrEmptyScheduleName = errors.New("empty schedule name provided") + + // ErrEmptyScheduleRepoID defines the error type when a Schedule type has an empty RepoID field provided. + ErrEmptyScheduleRepoID = errors.New("empty schedule repo_id provided") + + // ErrInvalidScheduleEntry defines the error type when a Schedule type has an invalid Entry field provided. + ErrInvalidScheduleEntry = errors.New("invalid schedule entry provided") +) + +type Schedule struct { + ID sql.NullInt64 `sql:"id"` + RepoID sql.NullInt64 `sql:"repo_id"` + Active sql.NullBool `sql:"active"` + Name sql.NullString `sql:"name"` + Entry sql.NullString `sql:"entry"` + CreatedAt sql.NullInt64 `sql:"created_at"` + CreatedBy sql.NullString `sql:"created_by"` + UpdatedAt sql.NullInt64 `sql:"updated_at"` + UpdatedBy sql.NullString `sql:"updated_by"` + ScheduledAt sql.NullInt64 `sql:"scheduled_at"` + Branch sql.NullString `sql:"branch"` + Error sql.NullString `sql:"error"` + + Repo Repo `gorm:"foreignKey:RepoID"` +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Schedule type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (s *Schedule) Nullify() *Schedule { + if s == nil { + return nil + } + + // check if the ID field should be valid + s.ID.Valid = s.ID.Int64 != 0 + // check if the RepoID field should be valid + s.RepoID.Valid = s.RepoID.Int64 != 0 + // check if the ID field should be valid + s.Active.Valid = s.RepoID.Int64 != 0 + // check if the Name field should be valid + s.Name.Valid = len(s.Name.String) != 0 + // check if the Entry field should be valid + s.Entry.Valid = len(s.Entry.String) != 0 + // check if the CreatedAt field should be valid + s.CreatedAt.Valid = s.CreatedAt.Int64 != 0 + // check if the CreatedBy field should be valid + s.CreatedBy.Valid = len(s.CreatedBy.String) != 0 + // check if the UpdatedAt field should be valid + s.UpdatedAt.Valid = s.UpdatedAt.Int64 != 0 + // check if the UpdatedBy field should be valid + s.UpdatedBy.Valid = len(s.UpdatedBy.String) != 0 + // check if the ScheduledAt field should be valid + s.ScheduledAt.Valid = s.ScheduledAt.Int64 != 0 + // check if the Branch field should be valid + s.Branch.Valid = len(s.Branch.String) != 0 + // check if the Error field should be valid + s.Error.Valid = len(s.Error.String) != 0 + + return s +} + +// ToAPI converts the Schedule type +// to an API Schedule type. +func (s *Schedule) ToAPI() *api.Schedule { + schedule := new(api.Schedule) + + schedule.SetID(s.ID.Int64) + schedule.SetRepo(s.Repo.ToAPI()) + schedule.SetActive(s.Active.Bool) + schedule.SetName(s.Name.String) + schedule.SetEntry(s.Entry.String) + schedule.SetCreatedAt(s.CreatedAt.Int64) + schedule.SetCreatedBy(s.CreatedBy.String) + schedule.SetUpdatedAt(s.UpdatedAt.Int64) + schedule.SetUpdatedBy(s.UpdatedBy.String) + schedule.SetScheduledAt(s.ScheduledAt.Int64) + schedule.SetBranch(s.Branch.String) + schedule.SetError(s.Error.String) + + return schedule +} + +// Validate verifies the necessary fields for +// the Schedule type are populated correctly. +func (s *Schedule) Validate() error { + // verify the RepoID field is populated + if s.RepoID.Int64 <= 0 { + return ErrEmptyScheduleRepoID + } + + // verify the Name field is populated + if len(s.Name.String) <= 0 { + return ErrEmptyScheduleName + } + + // verify the Entry field is populated + if len(s.Entry.String) <= 0 { + return ErrEmptyScheduleEntry + } + + gron := gronx.New() + if !gron.IsValid(s.Entry.String) { + return ErrInvalidScheduleEntry + } + + // ensure that all Schedule string fields that can be returned as JSON are sanitized to avoid unsafe HTML content + s.Name = sql.NullString{String: util.Sanitize(s.Name.String), Valid: s.Name.Valid} + s.Entry = sql.NullString{String: util.Sanitize(s.Entry.String), Valid: s.Entry.Valid} + + return nil +} + +// ScheduleFromAPI converts the API Schedule type +// to a database Schedule type. +func ScheduleFromAPI(s *api.Schedule) *Schedule { + schedule := &Schedule{ + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + RepoID: sql.NullInt64{Int64: s.GetRepo().GetID(), Valid: true}, + Active: sql.NullBool{Bool: s.GetActive(), Valid: true}, + Name: sql.NullString{String: s.GetName(), Valid: true}, + Entry: sql.NullString{String: s.GetEntry(), Valid: true}, + CreatedAt: sql.NullInt64{Int64: s.GetCreatedAt(), Valid: true}, + CreatedBy: sql.NullString{String: s.GetCreatedBy(), Valid: true}, + UpdatedAt: sql.NullInt64{Int64: s.GetUpdatedAt(), Valid: true}, + UpdatedBy: sql.NullString{String: s.GetUpdatedBy(), Valid: true}, + ScheduledAt: sql.NullInt64{Int64: s.GetScheduledAt(), Valid: true}, + Branch: sql.NullString{String: s.GetBranch(), Valid: true}, + Error: sql.NullString{String: s.GetError(), Valid: true}, + } + + return schedule.Nullify() +} diff --git a/database/types/schedule_test.go b/database/types/schedule_test.go new file mode 100644 index 000000000..dcf056218 --- /dev/null +++ b/database/types/schedule_test.go @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "reflect" + "testing" + "time" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/testutils" + "github.com/go-vela/types/constants" +) + +func TestTypes_Schedule_Nullify(t *testing.T) { + tests := []struct { + name string + schedule *Schedule + want *Schedule + }{ + { + name: "schedule with fields", + schedule: testSchedule(), + want: testSchedule(), + }, + { + name: "schedule with empty fields", + schedule: new(Schedule), + want: &Schedule{ + ID: sql.NullInt64{Int64: 0, Valid: false}, + RepoID: sql.NullInt64{Int64: 0, Valid: false}, + Active: sql.NullBool{Bool: false, Valid: false}, + Name: sql.NullString{String: "", Valid: false}, + Entry: sql.NullString{String: "", Valid: false}, + CreatedAt: sql.NullInt64{Int64: 0, Valid: false}, + CreatedBy: sql.NullString{String: "", Valid: false}, + UpdatedAt: sql.NullInt64{Int64: 0, Valid: false}, + UpdatedBy: sql.NullString{String: "", Valid: false}, + ScheduledAt: sql.NullInt64{Int64: 0, Valid: false}, + Branch: sql.NullString{String: "", Valid: false}, + Error: sql.NullString{String: "", Valid: false}, + }, + }, + { + name: "empty schedule", + schedule: nil, + want: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := test.schedule.Nullify() + if !reflect.DeepEqual(got, test.want) { + t.Errorf("Nullify is %v, want %v", got, test.want) + } + }) + } +} + +func TestTypes_Schedule_ToAPI(t *testing.T) { + // setup types + e := api.NewEventsFromMask(1) + + owner := testutils.APIUser().Crop() + owner.SetID(1) + owner.SetName("octocat") + owner.SetActive(true) + owner.SetToken("superSecretToken") + owner.SetRefreshToken("superSecretRefreshToken") + + repo := testutils.APIRepo() + repo.SetID(1) + repo.SetOwner(owner) + repo.SetHash("superSecretHash") + repo.SetOrg("github") + repo.SetName("octocat") + repo.SetFullName("github/octocat") + repo.SetLink("https://github.com/github/octocat") + repo.SetClone("https://github.com/github/octocat.git") + repo.SetBranch("main") + repo.SetTopics([]string{"cloud", "security"}) + repo.SetBuildLimit(10) + repo.SetTimeout(30) + repo.SetCounter(0) + repo.SetVisibility("public") + repo.SetPrivate(false) + repo.SetTrusted(false) + repo.SetActive(true) + repo.SetAllowEvents(e) + repo.SetPipelineType("yaml") + repo.SetPreviousName("oldName") + repo.SetApproveBuild(constants.ApproveNever) + + want := testutils.APISchedule() + want.SetID(1) + want.SetActive(true) + want.SetRepo(repo) + want.SetName("nightly") + want.SetEntry("0 0 * * *") + want.SetCreatedAt(time.Now().UTC().Unix()) + want.SetCreatedBy("user1") + want.SetUpdatedAt(time.Now().Add(time.Hour * 1).UTC().Unix()) + want.SetUpdatedBy("user2") + want.SetScheduledAt(time.Now().Add(time.Hour * 2).UTC().Unix()) + want.SetBranch("main") + want.SetError("unable to trigger build for schedule nightly: unknown character") + + // run test + got := testSchedule().ToAPI() + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestTypes_Schedule_Validate(t *testing.T) { + tests := []struct { + name string + failure bool + schedule *Schedule + }{ + { + name: "schedule with valid fields", + failure: false, + schedule: testSchedule(), + }, + { + name: "schedule with invalid entry", + failure: true, + schedule: &Schedule{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + RepoID: sql.NullInt64{Int64: 1, Valid: true}, + Name: sql.NullString{String: "invalid", Valid: false}, + Entry: sql.NullString{String: "!@#$%^&*()", Valid: false}, + }, + }, + { + name: "schedule with missing entry", + failure: true, + schedule: &Schedule{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + RepoID: sql.NullInt64{Int64: 1, Valid: true}, + Name: sql.NullString{String: "nightly", Valid: false}, + }, + }, + { + name: "schedule with missing name", + failure: true, + schedule: &Schedule{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + RepoID: sql.NullInt64{Int64: 1, Valid: true}, + Entry: sql.NullString{String: "0 0 * * *", Valid: false}, + }, + }, + { + name: "schedule with missing repo_id", + failure: true, + schedule: &Schedule{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Name: sql.NullString{String: "nightly", Valid: false}, + Entry: sql.NullString{String: "0 0 * * *", Valid: false}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.schedule.Validate() + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + return + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + }) + } +} + +func TestTypes_ScheduleFromAPI(t *testing.T) { + // setup types + s := new(api.Schedule) + repo := testutils.APIRepo() + repo.SetID(1) + + s.SetID(1) + s.SetActive(true) + s.SetRepo(repo) + s.SetName("nightly") + s.SetEntry("0 0 * * *") + s.SetCreatedAt(time.Now().UTC().Unix()) + s.SetCreatedBy("user1") + s.SetUpdatedAt(time.Now().Add(time.Hour * 1).UTC().Unix()) + s.SetUpdatedBy("user2") + s.SetScheduledAt(time.Now().Add(time.Hour * 2).UTC().Unix()) + s.SetBranch("main") + s.SetError("unable to trigger build for schedule nightly: unknown character") + + want := testSchedule() + want.Repo = Repo{} + + // run test + got := ScheduleFromAPI(s) + + if !reflect.DeepEqual(got, want) { + t.Errorf("ScheduleFromAPI is %v, want %v", got, want) + } +} + +// testSchedule is a test helper function to create a Schedule type with all fields set to a fake value. +func testSchedule() *Schedule { + return &Schedule{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + RepoID: sql.NullInt64{Int64: 1, Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + Name: sql.NullString{String: "nightly", Valid: true}, + Entry: sql.NullString{String: "0 0 * * *", Valid: true}, + CreatedAt: sql.NullInt64{Int64: time.Now().UTC().Unix(), Valid: true}, + CreatedBy: sql.NullString{String: "user1", Valid: true}, + UpdatedAt: sql.NullInt64{Int64: time.Now().Add(time.Hour * 1).UTC().Unix(), Valid: true}, + UpdatedBy: sql.NullString{String: "user2", Valid: true}, + ScheduledAt: sql.NullInt64{Int64: time.Now().Add(time.Hour * 2).UTC().Unix(), Valid: true}, + Branch: sql.NullString{String: "main", Valid: true}, + Error: sql.NullString{String: "unable to trigger build for schedule nightly: unknown character", Valid: true}, + + Repo: *testRepo(), + } +} diff --git a/mock/server/schedule.go b/mock/server/schedule.go index 417890cd7..390aa6b4a 100644 --- a/mock/server/schedule.go +++ b/mock/server/schedule.go @@ -10,28 +10,120 @@ import ( "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" - "github.com/go-vela/types/library" ) const ( // ScheduleResp represents a JSON return for a single schedule. ScheduleResp = `{ - "id": 2, - "repo_id": 1, - "active": true, - "name": "foo", - "entry": "@weekly", - "created_at": 1683154980, - "created_by": "octocat", - "updated_at": 1683154980, - "updated_by": "octocat", - "scheduled_at": 0, - "branch": "main" -}` + "id": 2, + "repo": { + "id": 1, + "owner": { + "id": 1, + "name": "octocat", + "favorites": [], + "active": true, + "admin": false + }, + "org": "github", + "counter": 10, + "name": "octocat", + "full_name": "github/octocat", + "link": "https://github.com/github/octocat", + "clone": "https://github.com/github/octocat", + "branch": "main", + "build_limit": 10, + "timeout": 60, + "visibility": "public", + "private": false, + "trusted": true, + "pipeline_type": "yaml", + "topics": [], + "active": true, + "allow_events": { + "push": { + "branch": true, + "tag": true + }, + "pull_request": { + "opened": true, + "synchronize": true, + "reopened": true, + "edited": false + }, + "deployment": { + "created": true + }, + "comment": { + "created": false, + "edited": false + } + }, + "approve_build": "fork-always", + "previous_name": "" + }, + "active": true, + "name": "foo", + "entry": "@weekly", + "created_at": 1683154980, + "created_by": "octocat", + "updated_at": 1683154980, + "updated_by": "octocat", + "scheduled_at": 0, + "branch": "main", + "error": "error message" + }` SchedulesResp = `[ { "id": 2, + "repo": { + "id": 1, + "owner": { + "id": 1, + "name": "octocat", + "favorites": [], + "active": true, + "admin": false + }, + "org": "github", + "counter": 10, + "name": "octocat", + "full_name": "github/octocat", + "link": "https://github.com/github/octocat", + "clone": "https://github.com/github/octocat", + "branch": "main", + "build_limit": 10, + "timeout": 60, + "visibility": "public", + "private": false, + "trusted": true, + "pipeline_type": "yaml", + "topics": [], + "active": true, + "allow_events": { + "push": { + "branch": true, + "tag": true + }, + "pull_request": { + "opened": true, + "synchronize": true, + "reopened": true, + "edited": false + }, + "deployment": { + "created": true + }, + "comment": { + "created": false, + "edited": false + } + }, + "approve_build": "fork-always", + "previous_name": "" + }, "active": true, "name": "foo", "entry": "@weekly", @@ -40,11 +132,57 @@ const ( "updated_at": 1683154980, "updated_by": "octocat", "scheduled_at": 0, - "repo_id": 1, - "branch": "main" + "branch": "main", + "error": "error message" }, { "id": 1, + "repo": { + "id": 1, + "owner": { + "id": 1, + "name": "octocat", + "favorites": [], + "active": true, + "admin": false + }, + "org": "github", + "counter": 10, + "name": "octocat", + "full_name": "github/octocat", + "link": "https://github.com/github/octocat", + "clone": "https://github.com/github/octocat", + "branch": "main", + "build_limit": 10, + "timeout": 60, + "visibility": "public", + "private": false, + "trusted": true, + "pipeline_type": "yaml", + "topics": [], + "active": true, + "allow_events": { + "push": { + "branch": true, + "tag": true + }, + "pull_request": { + "opened": true, + "synchronize": true, + "reopened": true, + "edited": false + }, + "deployment": { + "created": true + }, + "comment": { + "created": false, + "edited": false + } + }, + "approve_build": "fork-always", + "previous_name": "" + }, "active": true, "name": "bar", "entry": "@weekly", @@ -54,16 +192,16 @@ const ( "updated_by": "octocat", "scheduled_at": 0, "repo_id": 1, - "branch": "main - } -]` + "branch": "main", + "error": "error message" + }]` ) // getSchedules returns mock JSON for a http GET. func getSchedules(c *gin.Context) { data := []byte(SchedulesResp) - var body []library.Schedule + var body []api.Schedule _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -85,7 +223,7 @@ func getSchedule(c *gin.Context) { data := []byte(ScheduleResp) - var body library.Schedule + var body api.Schedule _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -95,7 +233,7 @@ func getSchedule(c *gin.Context) { func addSchedule(c *gin.Context) { data := []byte(ScheduleResp) - var body library.Schedule + var body api.Schedule _ = json.Unmarshal(data, &body) c.JSON(http.StatusCreated, body) @@ -119,7 +257,7 @@ func updateSchedule(c *gin.Context) { data := []byte(ScheduleResp) - var body library.Schedule + var body api.Schedule _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) diff --git a/mock/server/schedule_test.go b/mock/server/schedule_test.go index b88c9f223..293676e02 100644 --- a/mock/server/schedule_test.go +++ b/mock/server/schedule_test.go @@ -7,11 +7,11 @@ import ( "reflect" "testing" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestSchedule_ActiveScheduleResp(t *testing.T) { - testSchedule := library.Schedule{} + testSchedule := api.Schedule{} err := json.Unmarshal([]byte(ScheduleResp), &testSchedule) if err != nil { diff --git a/router/middleware/schedule/context.go b/router/middleware/schedule/context.go index 258c243ca..5c8920d01 100644 --- a/router/middleware/schedule/context.go +++ b/router/middleware/schedule/context.go @@ -5,7 +5,7 @@ package schedule import ( "context" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) const key = "schedule" @@ -16,13 +16,13 @@ type Setter interface { } // FromContext returns the Schedule associated with this context. -func FromContext(c context.Context) *library.Schedule { +func FromContext(c context.Context) *api.Schedule { value := c.Value(key) if value == nil { return nil } - s, ok := value.(*library.Schedule) + s, ok := value.(*api.Schedule) if !ok { return nil } @@ -32,6 +32,6 @@ func FromContext(c context.Context) *library.Schedule { // ToContext adds the Schedule to this context if it supports // the Setter interface. -func ToContext(c Setter, s *library.Schedule) { +func ToContext(c Setter, s *api.Schedule) { c.Set(key, s) } diff --git a/router/middleware/schedule/context_test.go b/router/middleware/schedule/context_test.go index 066a3a6cc..f0f841135 100644 --- a/router/middleware/schedule/context_test.go +++ b/router/middleware/schedule/context_test.go @@ -7,13 +7,13 @@ import ( "github.com/gin-gonic/gin" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestSchedule_FromContext(t *testing.T) { // setup types num := int64(1) - want := &library.Schedule{ID: &num} + want := &api.Schedule{ID: &num} // setup context gin.SetMode(gin.TestMode) @@ -72,7 +72,7 @@ func TestSchedule_FromContext_Empty(t *testing.T) { func TestSchedule_ToContext(t *testing.T) { // setup types num := int64(1) - want := &library.Schedule{ID: &num} + want := &api.Schedule{ID: &num} // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/schedule/schedule.go b/router/middleware/schedule/schedule.go index 78c5f14e5..41864cef0 100644 --- a/router/middleware/schedule/schedule.go +++ b/router/middleware/schedule/schedule.go @@ -9,15 +9,15 @@ import ( "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" ) // Retrieve gets the schedule in the given context. -func Retrieve(c *gin.Context) *library.Schedule { +func Retrieve(c *gin.Context) *api.Schedule { return FromContext(c) } From 76400eb9b8c208c880497cb3adc126b243649e78 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Wed, 8 May 2024 14:19:56 -0400 Subject: [PATCH 06/14] fix(ci): remove deprecated linter rules (#1123) --- .golangci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 5b27708f8..f857a2c9f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -71,7 +71,6 @@ linters: - bidichk # checks for dangerous unicode character sequences - bodyclose # checks whether HTTP response body is closed successfully - contextcheck # check the function whether use a non-inherited context - - deadcode # finds unused code - dupl # code clone detection - errcheck # checks for unchecked errors - errorlint # find misuses of errors @@ -97,14 +96,12 @@ linters: - nolintlint # reports ill-formed or insufficient nolint directives - revive # linter for go - staticcheck # applies static analysis checks, go vet on steroids - - structcheck # finds unused struct fields - stylecheck # replacement for golint - tenv # analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 - typecheck # parses and type-checks go code, like the front-end of a go compiler - unconvert # remove unnecessary type conversions - unparam # reports unused function parameters - unused # checks for unused constants, variables, functions and types - - varcheck # finds unused global variables and constants - whitespace # detects leading and trailing whitespace - wsl # forces code to use empty lines From 8a744e48d224197478bb48a468186783b04c44db Mon Sep 17 00:00:00 2001 From: David May <49894298+wass3rw3rk@users.noreply.github.com> Date: Wed, 8 May 2024 16:04:06 -0500 Subject: [PATCH 07/14] fix(api-spec): add missing 404 responses in docs (#1124) --- api/dashboard/delete.go | 4 ++++ api/dashboard/get.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/api/dashboard/delete.go b/api/dashboard/delete.go index 14e81ffa5..a2798c940 100644 --- a/api/dashboard/delete.go +++ b/api/dashboard/delete.go @@ -40,6 +40,10 @@ import ( // description: Unauthorized to delete dashboard // schema: // "$ref": "#/definitions/Error" +// '404': +// description: Unable to find dashboard +// schema: +// "$ref": "#/definitions/Error" // '500': // description: Server error when deleting dashboard // schema: diff --git a/api/dashboard/get.go b/api/dashboard/get.go index da4239667..3535cb8b4 100644 --- a/api/dashboard/get.go +++ b/api/dashboard/get.go @@ -42,6 +42,10 @@ import ( // description: Unauthorized to retrieve dashboard // schema: // "$ref": "#/definitions/Error" +// '404': +// description: Unable to find dashboard +// schema: +// "$ref": "#/definitions/Error" // '500': // description: Server error when retrieving dashboard // schema: From 4421e304b842c050e6260c0e563bb15b4a3066e6 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Fri, 10 May 2024 11:56:05 -0500 Subject: [PATCH 08/14] fix(compiler): aggregate templates for nested pipelines (#1125) --- compiler/native/expand.go | 4 ++++ compiler/native/expand_test.go | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/compiler/native/expand.go b/compiler/native/expand.go index e69066ef7..335ca5007 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -60,6 +60,7 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * secrets := s.Secrets services := s.Services environment := s.Environment + templates := s.Templates if len(environment) == 0 { environment = make(raw.StringSliceMap) @@ -139,6 +140,8 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * return s, fmt.Errorf("cannot use render_inline inside a called template (%s)", step.Template.Name) } + templates = append(templates, tmplBuild.Templates...) + tmplBuild, err = c.ExpandSteps(tmplBuild, mapFromTemplates(tmplBuild.Templates), r, depth-1) if err != nil { return s, err @@ -202,6 +205,7 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * s.Secrets = secrets s.Services = services s.Environment = environment + s.Templates = templates return s, nil } diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index cf6f3927f..0e69eed65 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -869,6 +869,19 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { "star": "test3", } + wantTemplates := yaml.TemplateSlice{ + { + Name: "chain", + Source: "github.example.com/faz/baz/template_calls_template.yml", + Type: "github", + }, + { + Name: "test", + Source: "github.example.com/foo/bar/long_template.yml", + Type: "github", + }, + } + // run test compiler, err := New(c) if err != nil { @@ -879,7 +892,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment, Templates: yaml.TemplateSlice{test.tmpls["chain"]}}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -899,6 +912,10 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { if diff := cmp.Diff(build.Environment, wantEnvironment); diff != "" { t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) } + + if diff := cmp.Diff(build.Templates, wantTemplates); diff != "" { + t.Errorf("ExpandSteps()_Type%s mismatch (-want +got):\n%s", test.name, diff) + } }) } } From 27f116370e14814a0403c61b829c6cd70f3c8e13 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Fri, 10 May 2024 14:38:12 -0500 Subject: [PATCH 09/14] feat!: db driven platform settings (#1110) --- api/admin/settings.go | 278 +++++++++++++++++ api/repo/create.go | 7 +- api/schedule/create.go | 36 +-- api/types/settings/compiler.go | 112 +++++++ api/types/settings/compiler_test.go | 116 +++++++ api/types/settings/platform.go | 285 ++++++++++++++++++ api/types/settings/platform_test.go | 195 ++++++++++++ api/types/settings/queue.go | 52 ++++ api/types/settings/queue_test.go | 90 ++++++ cmd/vela-server/compiler.go | 24 -- cmd/vela-server/main.go | 6 + cmd/vela-server/queue.go | 31 -- cmd/vela-server/schedule.go | 9 +- cmd/vela-server/server.go | 97 +++++- compiler/engine.go | 9 +- compiler/native/clone.go | 4 +- compiler/native/clone_test.go | 4 +- compiler/native/compile.go | 10 +- compiler/native/compile_test.go | 34 ++- compiler/native/environment_test.go | 12 +- compiler/native/expand.go | 6 +- compiler/native/expand_test.go | 35 ++- compiler/native/initialize_test.go | 6 +- compiler/native/native.go | 28 +- compiler/native/native_test.go | 73 +++-- compiler/native/parse.go | 2 +- compiler/native/parse_test.go | 24 +- compiler/native/script_test.go | 6 +- compiler/native/settings.go | 21 ++ compiler/native/substitute_test.go | 6 +- compiler/native/transform_test.go | 6 +- compiler/native/validate_test.go | 54 ++-- database/database.go | 2 + database/integration_test.go | 55 ++++ database/interface.go | 4 + database/resource.go | 14 + database/resource_test.go | 3 + database/secret/create_test.go | 7 +- database/secret/secret_test.go | 28 -- database/secret/update_test.go | 7 +- database/settings/create.go | 32 ++ database/settings/create_test.go | 82 +++++ database/settings/get.go | 31 ++ database/settings/get_test.go | 92 ++++++ database/settings/interface.go | 22 ++ database/settings/opts.go | 52 ++++ database/settings/opts_test.go | 208 +++++++++++++ database/settings/settings.go | 77 +++++ database/settings/settings_test.go | 171 +++++++++++ database/settings/table.go | 60 ++++ database/settings/table_test.go | 58 ++++ database/settings/update.go | 34 +++ database/settings/update_test.go | 87 ++++++ database/testutils/mock_args.go | 35 +++ database/types/settings.go | 218 ++++++++++++++ database/types/settings_test.go | 195 ++++++++++++ go.mod | 2 + go.sum | 4 + internal/image/doc.go | 9 + internal/image/image.go | 35 +++ internal/image/image_test.go | 76 +++++ mock/server/server.go | 3 + mock/server/settings.go | 119 ++++++++ mock/server/settings_test.go | 65 ++++ queue/queue.go | 22 ++ queue/redis/driver_test.go | 2 +- queue/redis/length.go | 6 +- queue/redis/length_test.go | 18 +- queue/redis/opts.go | 16 +- queue/redis/opts_test.go | 30 +- queue/redis/ping_test.go | 4 +- queue/redis/pop.go | 14 +- queue/redis/pop_test.go | 20 +- queue/redis/redis.go | 11 +- queue/redis/redis_test.go | 2 +- queue/redis/route.go | 4 +- queue/redis/settings.go | 19 ++ queue/service.go | 9 + queue/setup.go | 2 +- router/admin.go | 32 +- router/middleware/allowlist.go | 16 - router/middleware/allowlist_schedule.go | 16 - router/middleware/cli.go | 20 ++ router/middleware/cli/context.go | 37 +++ router/middleware/cli/context_test.go | 85 ++++++ router/middleware/cli/doc.go | 10 + ...allowlist_schedule_test.go => cli_test.go} | 20 +- router/middleware/compiler.go | 9 +- router/middleware/compiler_test.go | 33 +- router/middleware/pipeline/pipeline_test.go | 2 +- router/middleware/queue.go | 5 + router/middleware/settings.go | 20 ++ router/middleware/settings/context.go | 37 +++ router/middleware/settings/context_test.go | 106 +++++++ router/middleware/settings/doc.go | 10 + .../{allowlist_test.go => settings_test.go} | 18 +- router/middleware/worker_test.go | 4 +- 97 files changed, 3860 insertions(+), 364 deletions(-) create mode 100644 api/admin/settings.go create mode 100644 api/types/settings/compiler.go create mode 100644 api/types/settings/compiler_test.go create mode 100644 api/types/settings/platform.go create mode 100644 api/types/settings/platform_test.go create mode 100644 api/types/settings/queue.go create mode 100644 api/types/settings/queue_test.go delete mode 100644 cmd/vela-server/compiler.go delete mode 100644 cmd/vela-server/queue.go create mode 100644 compiler/native/settings.go create mode 100644 database/settings/create.go create mode 100644 database/settings/create_test.go create mode 100644 database/settings/get.go create mode 100644 database/settings/get_test.go create mode 100644 database/settings/interface.go create mode 100644 database/settings/opts.go create mode 100644 database/settings/opts_test.go create mode 100644 database/settings/settings.go create mode 100644 database/settings/settings_test.go create mode 100644 database/settings/table.go create mode 100644 database/settings/table_test.go create mode 100644 database/settings/update.go create mode 100644 database/settings/update_test.go create mode 100644 database/testutils/mock_args.go create mode 100644 database/types/settings.go create mode 100644 database/types/settings_test.go create mode 100644 internal/image/doc.go create mode 100644 internal/image/image.go create mode 100644 internal/image/image_test.go create mode 100644 mock/server/settings.go create mode 100644 mock/server/settings_test.go create mode 100644 queue/redis/settings.go delete mode 100644 router/middleware/allowlist.go delete mode 100644 router/middleware/allowlist_schedule.go create mode 100644 router/middleware/cli.go create mode 100644 router/middleware/cli/context.go create mode 100644 router/middleware/cli/context_test.go create mode 100644 router/middleware/cli/doc.go rename router/middleware/{allowlist_schedule_test.go => cli_test.go} (65%) create mode 100644 router/middleware/settings.go create mode 100644 router/middleware/settings/context.go create mode 100644 router/middleware/settings/context_test.go create mode 100644 router/middleware/settings/doc.go rename router/middleware/{allowlist_test.go => settings_test.go} (60%) diff --git a/api/admin/settings.go b/api/admin/settings.go new file mode 100644 index 000000000..0b067372b --- /dev/null +++ b/api/admin/settings.go @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: Apache-2.0 + +package admin + +import ( + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/compiler/native" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal/image" + "github.com/go-vela/server/queue" + cliMiddleware "github.com/go-vela/server/router/middleware/cli" + sMiddleware "github.com/go-vela/server/router/middleware/settings" + uMiddleware "github.com/go-vela/server/router/middleware/user" + "github.com/go-vela/server/util" +) + +// swagger:operation GET /api/v1/admin/settings admin GetSettings +// +// Get the currently configured settings. +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully retrieved settings +// type: json +// schema: +// "$ref": "#/definitions/Platform" +// '404': +// description: Unable to retrieve settings +// schema: +// "$ref": "#/definitions/Error" + +// GetSettings represents the API handler to +// captures settings stored in the database. +func GetSettings(c *gin.Context) { + // capture middleware values + s := sMiddleware.FromContext(c) + + logrus.Info("Admin: reading settings") + + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + c.JSON(http.StatusOK, s) +} + +// swagger:operation PUT /api/v1/admin/settings admin UpdateSettings +// +// Update the platform settings singleton in the database. +// +// --- +// produces: +// - application/json +// parameters: +// - in: body +// name: body +// description: Payload containing settings to update +// required: true +// schema: +// "$ref": "#/definitions/Platform" +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully updated platform settings in the database +// type: json +// schema: +// "$ref": "#/definitions/Platform" +// '400': +// description: Unable to update settings — bad request +// schema: +// "$ref": "#/definitions/Error" +// '404': +// description: Unable to retrieve platform settings to update +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to update platform settings in the database +// schema: +// "$ref": "#/definitions/Error" + +// UpdateSettings represents the API handler to +// update the settings singleton stored in the database. +func UpdateSettings(c *gin.Context) { + // capture middleware values + s := sMiddleware.FromContext(c) + u := uMiddleware.FromContext(c) + ctx := c.Request.Context() + + logrus.Info("Admin: updating settings") + + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + // duplicate settings to not alter the shared pointer + _s := new(settings.Platform) + _s.Update(s) + + // ensure we update the singleton record + _s.SetID(1) + + // capture body from API request + input := new(settings.Platform) + + err := c.Bind(input) + if err != nil { + retErr := fmt.Errorf("unable to decode JSON for settings: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + if input.Compiler != nil { + if input.CloneImage != nil { + // validate clone image + cloneImage := *input.CloneImage + + _, err = image.ParseWithError(cloneImage) + if err != nil { + retErr := fmt.Errorf("invalid clone image %s: %w", cloneImage, err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + _s.SetCloneImage(cloneImage) + } + + if input.TemplateDepth != nil { + _s.SetTemplateDepth(*input.TemplateDepth) + } + + if input.StarlarkExecLimit != nil { + _s.SetStarlarkExecLimit(*input.StarlarkExecLimit) + } + } + + if input.Queue != nil { + if input.Queue.Routes != nil { + _s.SetRoutes(input.GetRoutes()) + } + } + + if input.RepoAllowlist != nil { + _s.SetRepoAllowlist(input.GetRepoAllowlist()) + } + + if input.ScheduleAllowlist != nil { + _s.SetScheduleAllowlist(input.GetScheduleAllowlist()) + } + + _s.SetUpdatedBy(u.GetName()) + + // send API call to update the settings + _s, err = database.FromContext(c).UpdateSettings(ctx, _s) + if err != nil { + retErr := fmt.Errorf("unable to update settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, _s) +} + +// swagger:operation DELETE /api/v1/admin/settings admin RestoreSettings +// +// Restore the currently configured settings to the environment defaults. +// +// --- +// produces: +// - application/json +// security: +// - ApiKeyAuth: [] +// responses: +// '200': +// description: Successfully restored default settings in the database +// type: json +// schema: +// "$ref": "#/definitions/Platform" +// '404': +// description: Unable to retrieve settings to restore +// schema: +// "$ref": "#/definitions/Error" +// '500': +// description: Unable to restore settings in the database +// schema: +// "$ref": "#/definitions/Error" + +// RestoreSettings represents the API handler to +// restore settings stored in the database to the environment defaults. +func RestoreSettings(c *gin.Context) { + // capture middleware values + s := sMiddleware.FromContext(c) + u := uMiddleware.FromContext(c) + cliCtx := cliMiddleware.FromContext(c) + ctx := c.Request.Context() + + logrus.Info("Admin: restoring settings") + + // check captured value because we aren't retrieving settings from the database + // instead we are retrieving the auto-refreshed middleware value + if s == nil { + retErr := fmt.Errorf("settings not found") + + util.HandleError(c, http.StatusNotFound, retErr) + + return + } + + compiler, err := native.FromCLIContext(cliCtx) + if err != nil { + retErr := fmt.Errorf("unable to restore settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + queue, err := queue.FromCLIContext(cliCtx) + if err != nil { + retErr := fmt.Errorf("unable to restore settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + s.SetUpdatedAt(time.Now().UTC().Unix()) + s.SetUpdatedBy(u.GetName()) + + // read in defaults supplied from the cli runtime + compilerSettings := compiler.GetSettings() + s.SetCompiler(compilerSettings) + + queueSettings := queue.GetSettings() + s.SetQueue(queueSettings) + + // send API call to update the settings + s, err = database.FromContext(c).UpdateSettings(ctx, s) + if err != nil { + retErr := fmt.Errorf("unable to update (restore) settings: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.JSON(http.StatusOK, s) +} diff --git a/api/repo/create.go b/api/repo/create.go index 824f7d0f5..7249bbfa4 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -15,6 +15,7 @@ import ( "github.com/go-vela/server/api/types" "github.com/go-vela/server/api/types/actions" "github.com/go-vela/server/database" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/scm" "github.com/go-vela/server/util" @@ -71,7 +72,8 @@ import ( func CreateRepo(c *gin.Context) { // capture middleware values u := user.Retrieve(c) - allowlist := c.Value("allowlist").([]string) + s := settings.FromContext(c) + defaultBuildLimit := c.Value("defaultBuildLimit").(int64) defaultTimeout := c.Value("defaultTimeout").(int64) maxBuildLimit := c.Value("maxBuildLimit").(int64) @@ -197,6 +199,7 @@ func CreateRepo(c *gin.Context) { return } + r.SetPipelineType(input.GetPipelineType()) } @@ -217,7 +220,7 @@ func CreateRepo(c *gin.Context) { ) // ensure repo is allowed to be activated - if !util.CheckAllowlist(r, allowlist) { + if !util.CheckAllowlist(r, s.GetRepoAllowlist()) { retErr := fmt.Errorf("unable to activate repo: %s is not on allowlist", r.GetFullName()) util.HandleError(c, http.StatusForbidden, retErr) diff --git a/api/schedule/create.go b/api/schedule/create.go index a691230e6..a63d60fa4 100644 --- a/api/schedule/create.go +++ b/api/schedule/create.go @@ -14,6 +14,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/repo" + "github.com/go-vela/server/router/middleware/settings" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" ) @@ -77,7 +78,8 @@ func CreateSchedule(c *gin.Context) { u := user.Retrieve(c) r := repo.Retrieve(c) ctx := c.Request.Context() - allowlist := c.Value("allowlistschedule").([]string) + s := settings.FromContext(c) + minimumFrequency := c.Value("scheduleminimumfrequency").(time.Duration) // capture body from API request @@ -119,7 +121,7 @@ func CreateSchedule(c *gin.Context) { }).Infof("creating new schedule %s", input.GetName()) // ensure repo is allowed to create new schedules - if !util.CheckAllowlist(r, allowlist) { + if !util.CheckAllowlist(r, s.GetScheduleAllowlist()) { retErr := fmt.Errorf("unable to create schedule %s: %s is not on allowlist", input.GetName(), r.GetFullName()) util.HandleError(c, http.StatusForbidden, retErr) @@ -127,29 +129,29 @@ func CreateSchedule(c *gin.Context) { return } - s := new(api.Schedule) + schedule := new(api.Schedule) // update fields in schedule object - s.SetCreatedBy(u.GetName()) - s.SetRepo(r) - s.SetName(input.GetName()) - s.SetEntry(input.GetEntry()) - s.SetCreatedAt(time.Now().UTC().Unix()) - s.SetUpdatedAt(time.Now().UTC().Unix()) - s.SetUpdatedBy(u.GetName()) + schedule.SetCreatedBy(u.GetName()) + schedule.SetRepo(r) + schedule.SetName(input.GetName()) + schedule.SetEntry(input.GetEntry()) + schedule.SetCreatedAt(time.Now().UTC().Unix()) + schedule.SetUpdatedAt(time.Now().UTC().Unix()) + schedule.SetUpdatedBy(u.GetName()) if input.GetBranch() == "" { - s.SetBranch(r.GetBranch()) + schedule.SetBranch(r.GetBranch()) } else { - s.SetBranch(input.GetBranch()) + schedule.SetBranch(input.GetBranch()) } // set the active field based off the input provided if input.Active == nil { // default active field to true - s.SetActive(true) + schedule.SetActive(true) } else { - s.SetActive(input.GetActive()) + schedule.SetActive(input.GetActive()) } // send API call to capture the schedule from the database @@ -178,7 +180,7 @@ func CreateSchedule(c *gin.Context) { dbSchedule.SetActive(true) // send API call to update the schedule - s, err = database.FromContext(c).UpdateSchedule(ctx, dbSchedule, true) + schedule, err = database.FromContext(c).UpdateSchedule(ctx, dbSchedule, true) if err != nil { retErr := fmt.Errorf("unable to set schedule %s to active: %w", dbSchedule.GetName(), err) @@ -188,7 +190,7 @@ func CreateSchedule(c *gin.Context) { } } else { // send API call to create the schedule - s, err = database.FromContext(c).CreateSchedule(ctx, s) + schedule, err = database.FromContext(c).CreateSchedule(ctx, schedule) if err != nil { retErr := fmt.Errorf("unable to create new schedule %s: %w", r.GetName(), err) @@ -198,7 +200,7 @@ func CreateSchedule(c *gin.Context) { } } - c.JSON(http.StatusCreated, s) + c.JSON(http.StatusCreated, schedule) } // validateEntry validates the entry for a minimum frequency. diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go new file mode 100644 index 000000000..a18a31401 --- /dev/null +++ b/api/types/settings/compiler.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import "fmt" + +type Compiler struct { + CloneImage *string `json:"clone_image,omitempty"` + TemplateDepth *int `json:"template_depth,omitempty"` + StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` +} + +// GetCloneImage returns the CloneImage field. +// +// When the provided Compiler type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetCloneImage() string { + // return zero value if Settings type or CloneImage field is nil + if cs == nil || cs.CloneImage == nil { + return "" + } + + return *cs.CloneImage +} + +// GetTemplateDepth returns the TemplateDepth field. +// +// When the provided Compiler type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetTemplateDepth() int { + // return zero value if Settings type or TemplateDepth field is nil + if cs == nil || cs.TemplateDepth == nil { + return 0 + } + + return *cs.TemplateDepth +} + +// GetStarlarkExecLimit returns the StarlarkExecLimit field. +// +// When the provided Compiler type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (cs *Compiler) GetStarlarkExecLimit() uint64 { + // return zero value if Compiler type or StarlarkExecLimit field is nil + if cs == nil || cs.StarlarkExecLimit == nil { + return 0 + } + + return *cs.StarlarkExecLimit +} + +// SetCloneImage sets the CloneImage field. +// +// When the provided Compiler type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetCloneImage(v string) { + // return if Compiler type is nil + if cs == nil { + return + } + + cs.CloneImage = &v +} + +// SetTemplateDepth sets the TemplateDepth field. +// +// When the provided Compiler type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetTemplateDepth(v int) { + // return if Compiler type is nil + if cs == nil { + return + } + + cs.TemplateDepth = &v +} + +// SetStarlarkExecLimit sets the StarlarkExecLimit field. +// +// When the provided Compiler type is nil, it +// will set nothing and immediately return. +func (cs *Compiler) SetStarlarkExecLimit(v uint64) { + // return if Compiler type is nil + if cs == nil { + return + } + + cs.StarlarkExecLimit = &v +} + +// String implements the Stringer interface for the Compiler type. +func (cs *Compiler) String() string { + return fmt.Sprintf(`{ + CloneImage: %s, + TemplateDepth: %d, + StarlarkExecLimit: %d, +}`, + cs.GetCloneImage(), + cs.GetTemplateDepth(), + cs.GetStarlarkExecLimit(), + ) +} + +// CompilerMockEmpty returns an empty Compiler type. +func CompilerMockEmpty() Compiler { + cs := Compiler{} + cs.SetCloneImage("") + cs.SetTemplateDepth(0) + cs.SetStarlarkExecLimit(0) + + return cs +} diff --git a/api/types/settings/compiler_test.go b/api/types/settings/compiler_test.go new file mode 100644 index 000000000..a224a9cb7 --- /dev/null +++ b/api/types/settings/compiler_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTypes_Compiler_Getters(t *testing.T) { + // setup tests + tests := []struct { + compiler *Compiler + want *Compiler + }{ + { + compiler: testCompilerSettings(), + want: testCompilerSettings(), + }, + { + compiler: new(Compiler), + want: new(Compiler), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.compiler.GetCloneImage(), test.want.GetCloneImage()) { + t.Errorf("GetCloneImage is %v, want %v", test.compiler.GetCloneImage(), test.want.GetCloneImage()) + } + + if !reflect.DeepEqual(test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) { + t.Errorf("GetTemplateDepth is %v, want %v", test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) + } + + if !reflect.DeepEqual(test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) { + t.Errorf("GetStarlarkExecLimit is %v, want %v", test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) + } + } +} + +func TestTypes_Compiler_Setters(t *testing.T) { + // setup types + var cs *Compiler + + // setup tests + tests := []struct { + compiler *Compiler + want *Compiler + }{ + { + compiler: testCompilerSettings(), + want: testCompilerSettings(), + }, + { + compiler: cs, + want: new(Compiler), + }, + } + + // run tests + for _, test := range tests { + test.compiler.SetCloneImage(test.want.GetCloneImage()) + + if !reflect.DeepEqual(test.compiler.GetCloneImage(), test.want.GetCloneImage()) { + t.Errorf("SetCloneImage is %v, want %v", test.compiler.GetCloneImage(), test.want.GetCloneImage()) + } + + test.compiler.SetTemplateDepth(test.want.GetTemplateDepth()) + + if !reflect.DeepEqual(test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) { + t.Errorf("SetTemplateDepth is %v, want %v", test.compiler.GetTemplateDepth(), test.want.GetTemplateDepth()) + } + + test.compiler.SetStarlarkExecLimit(test.want.GetStarlarkExecLimit()) + + if !reflect.DeepEqual(test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) { + t.Errorf("SetStarlarkExecLimit is %v, want %v", test.compiler.GetStarlarkExecLimit(), test.want.GetStarlarkExecLimit()) + } + } +} + +func TestTypes_Compiler_String(t *testing.T) { + // setup types + cs := testCompilerSettings() + + want := fmt.Sprintf(`{ + CloneImage: %s, + TemplateDepth: %d, + StarlarkExecLimit: %d, +}`, + cs.GetCloneImage(), + cs.GetTemplateDepth(), + cs.GetStarlarkExecLimit(), + ) + + // run test + got := cs.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testCompilerSettings is a test helper function to create a Compiler +// type with all fields set to a fake value. +func testCompilerSettings() *Compiler { + cs := new(Compiler) + + cs.SetCloneImage("target/vela-git:latest") + cs.SetTemplateDepth(1) + cs.SetStarlarkExecLimit(100) + + return cs +} diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go new file mode 100644 index 000000000..f72698b66 --- /dev/null +++ b/api/types/settings/platform.go @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" +) + +// Platform is the API representation of platform settingps. +// +// swagger:model Platform +type Platform struct { + ID *int64 `json:"id"` + *Queue `json:"queue"` + *Compiler `json:"compiler"` + RepoAllowlist *[]string `json:"repo_allowlist"` + ScheduleAllowlist *[]string `json:"schedule_allowlist"` + CreatedAt *int64 `json:"created_at,omitempty"` + UpdatedAt *int64 `json:"updated_at,omitempty"` + UpdatedBy *string `json:"updated_by,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetID() int64 { + // return zero value if Platform type or ID field is nil + if ps == nil || ps.ID == nil { + return 0 + } + + return *ps.ID +} + +// GetCompiler returns the Compiler field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetCompiler() Compiler { + // return zero value if Platform type or Compiler field is nil + if ps == nil || ps.Compiler == nil { + return Compiler{} + } + + return *ps.Compiler +} + +// GetQueue returns the Queue field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetQueue() Queue { + // return zero value if Platform type or Queue field is nil + if ps == nil || ps.Queue == nil { + return Queue{} + } + + return *ps.Queue +} + +// GetRepoAllowlist returns the RepoAllowlist field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetRepoAllowlist() []string { + // return zero value if Platform type or RepoAllowlist field is nil + if ps == nil || ps.RepoAllowlist == nil { + return []string{} + } + + return *ps.RepoAllowlist +} + +// GetScheduleAllowlist returns the ScheduleAllowlist field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetScheduleAllowlist() []string { + // return zero value if Platform type or ScheduleAllowlist field is nil + if ps == nil || ps.ScheduleAllowlist == nil { + return []string{} + } + + return *ps.ScheduleAllowlist +} + +// GetCreatedAt returns the CreatedAt field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetCreatedAt() int64 { + // return zero value if Platform type or CreatedAt field is nil + if ps == nil || ps.CreatedAt == nil { + return 0 + } + + return *ps.CreatedAt +} + +// GetUpdatedAt returns the UpdatedAt field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetUpdatedAt() int64 { + // return zero value if Platform type or UpdatedAt field is nil + if ps == nil || ps.UpdatedAt == nil { + return 0 + } + + return *ps.UpdatedAt +} + +// GetUpdatedBy returns the UpdatedBy field. +// +// When the provided Platform type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (ps *Platform) GetUpdatedBy() string { + // return zero value if Platform type or UpdatedBy field is nil + if ps == nil || ps.UpdatedBy == nil { + return "" + } + + return *ps.UpdatedBy +} + +// SetID sets the ID field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetID(v int64) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.ID = &v +} + +// SetCompiler sets the Compiler field. +// +// When the provided Compiler type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetCompiler(cs Compiler) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.Compiler = &cs +} + +// SetQueue sets the Queue field. +// +// When the provided Queue type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetQueue(qs Queue) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.Queue = &qs +} + +// SetRepoAllowlist sets the RepoAllowlist field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetRepoAllowlist(v []string) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.RepoAllowlist = &v +} + +// SetScheduleAllowlist sets the RepoAllowlist field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetScheduleAllowlist(v []string) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.ScheduleAllowlist = &v +} + +// SetCreatedAt sets the CreatedAt field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetCreatedAt(v int64) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.CreatedAt = &v +} + +// SetUpdatedAt sets the UpdatedAt field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetUpdatedAt(v int64) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.UpdatedAt = &v +} + +// SetUpdatedBy sets the UpdatedBy field. +// +// When the provided Platform type is nil, it +// will set nothing and immediately return. +func (ps *Platform) SetUpdatedBy(v string) { + // return if Platform type is nil + if ps == nil { + return + } + + ps.UpdatedBy = &v +} + +// Update takes another settings record and updates the internal fields, intended +// to be used when the refreshing settings record shared across the server. +func (ps *Platform) Update(newSettingps *Platform) { + if ps == nil { + return + } + + if newSettingps == nil { + return + } + + ps.SetCompiler(newSettingps.GetCompiler()) + ps.SetQueue(newSettingps.GetQueue()) + ps.SetRepoAllowlist(newSettingps.GetRepoAllowlist()) + ps.SetScheduleAllowlist(newSettingps.GetScheduleAllowlist()) +} + +// String implements the Stringer interface for the Platform type. +func (ps *Platform) String() string { + cs := ps.GetCompiler() + qs := ps.GetQueue() + + return fmt.Sprintf(`{ + ID: %d, + Compiler: %v, + Queue: %v, + RepoAllowlist: %v, + ScheduleAllowlist: %v, + CreatedAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, +}`, + ps.GetID(), + cs.String(), + qs.String(), + ps.GetRepoAllowlist(), + ps.GetScheduleAllowlist(), + ps.GetCreatedAt(), + ps.GetUpdatedAt(), + ps.GetUpdatedBy(), + ) +} + +// PlatformMockEmpty returns an empty Platform type. +func PlatformMockEmpty() Platform { + ps := Platform{} + + ps.SetCompiler(CompilerMockEmpty()) + ps.SetQueue(QueueMockEmpty()) + + ps.SetRepoAllowlist([]string{}) + ps.SetScheduleAllowlist([]string{}) + + return ps +} diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go new file mode 100644 index 000000000..56c9cec7b --- /dev/null +++ b/api/types/settings/platform_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestTypes_Platform_Getters(t *testing.T) { + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: testPlatformSettings(), + want: testPlatformSettings(), + }, + { + platform: new(Platform), + want: new(Platform), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.platform.GetCompiler(), test.want.GetCompiler()) { + t.Errorf("GetCompiler is %v, want %v", test.platform.GetCompiler(), test.want.GetCompiler()) + } + + if !reflect.DeepEqual(test.platform.GetQueue(), test.want.GetQueue()) { + t.Errorf("GetQueue is %v, want %v", test.platform.GetQueue(), test.want.GetQueue()) + } + + if !reflect.DeepEqual(test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) { + t.Errorf("GetRepoAllowlist is %v, want %v", test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) + } + + if !reflect.DeepEqual(test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) { + t.Errorf("GetScheduleAllowlist is %v, want %v", test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) + } + } +} + +func TestTypes_Platform_Setters(t *testing.T) { + // setup types + var ps *Platform + + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: testPlatformSettings(), + want: testPlatformSettings(), + }, + { + platform: ps, + want: new(Platform), + }, + } + + // run tests + for _, test := range tests { + test.platform.SetCompiler(test.want.GetCompiler()) + + if !reflect.DeepEqual(test.platform.GetCompiler(), test.want.GetCompiler()) { + t.Errorf("SetCompiler is %v, want %v", test.platform.GetCompiler(), test.want.GetCompiler()) + } + + test.platform.SetQueue(test.want.GetQueue()) + + if !reflect.DeepEqual(test.platform.GetQueue(), test.want.GetQueue()) { + t.Errorf("SetQueue is %v, want %v", test.platform.GetQueue(), test.want.GetQueue()) + } + + test.platform.SetRepoAllowlist(test.want.GetRepoAllowlist()) + + if !reflect.DeepEqual(test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) { + t.Errorf("SetRepoAllowlist is %v, want %v", test.platform.GetRepoAllowlist(), test.want.GetRepoAllowlist()) + } + + test.platform.SetScheduleAllowlist(test.want.GetScheduleAllowlist()) + + if !reflect.DeepEqual(test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) { + t.Errorf("SetScheduleAllowlist is %v, want %v", test.platform.GetScheduleAllowlist(), test.want.GetScheduleAllowlist()) + } + } +} + +func TestTypes_Platform_Update(t *testing.T) { + // setup types + s := testPlatformSettings() + + // update fields + sUpdate := testPlatformSettings() + sUpdate.SetCompiler(Compiler{}) + sUpdate.SetQueue(Queue{}) + sUpdate.SetRepoAllowlist([]string{"foo"}) + sUpdate.SetScheduleAllowlist([]string{"bar"}) + + // setup tests + tests := []struct { + platform *Platform + want *Platform + }{ + { + platform: s, + want: testPlatformSettings(), + }, + { + platform: s, + want: sUpdate, + }, + } + + // run tests + for _, test := range tests { + test.platform.Update(test.want) + + if diff := cmp.Diff(test.want, test.platform); diff != "" { + t.Errorf("(Update: -want +got):\n%s", diff) + } + } +} + +func TestTypes_Platform_String(t *testing.T) { + // setup types + s := testPlatformSettings() + cs := s.GetCompiler() + qs := s.GetQueue() + + want := fmt.Sprintf(`{ + ID: %d, + Compiler: %v, + Queue: %v, + RepoAllowlist: %v, + ScheduleAllowlist: %v, + CreatedAt: %d, + UpdatedAt: %d, + UpdatedBy: %s, +}`, + s.GetID(), + cs.String(), + qs.String(), + s.GetRepoAllowlist(), + s.GetScheduleAllowlist(), + s.GetCreatedAt(), + s.GetUpdatedAt(), + s.GetUpdatedBy(), + ) + + // run test + got := s.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testPlatformSettings is a test helper function to create a Platform +// type with all fields set to a fake value. +func testPlatformSettings() *Platform { + // setup platform + s := new(Platform) + s.SetID(1) + s.SetCreatedAt(1) + s.SetUpdatedAt(1) + s.SetUpdatedBy("vela-server") + s.SetRepoAllowlist([]string{"foo", "bar"}) + s.SetScheduleAllowlist([]string{"*"}) + + // setup types + // setup compiler + cs := new(Compiler) + + cs.SetCloneImage("target/vela-git:latest") + cs.SetTemplateDepth(1) + cs.SetStarlarkExecLimit(100) + + // setup queue + qs := new(Queue) + + qs.SetRoutes([]string{"vela"}) + + s.SetCompiler(*cs) + s.SetQueue(*qs) + + return s +} diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go new file mode 100644 index 000000000..9bf54dfd1 --- /dev/null +++ b/api/types/settings/queue.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import "fmt" + +type Queue struct { + Routes *[]string `json:"routes,omitempty"` +} + +// GetRoutes returns the Routes field. +// +// When the provided Queue type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (qs *Queue) GetRoutes() []string { + // return zero value if Queue type or Routes field is nil + if qs == nil || qs.Routes == nil { + return []string{} + } + + return *qs.Routes +} + +// SetRoutes sets the Routes field. +// +// When the provided Queue type is nil, it +// will set nothing and immediately return. +func (qs *Queue) SetRoutes(v []string) { + // return if Queue type is nil + if qs == nil { + return + } + + qs.Routes = &v +} + +// String implements the Stringer interface for the Queue type. +func (qs *Queue) String() string { + return fmt.Sprintf(`{ + Routes: %v, +}`, + qs.GetRoutes(), + ) +} + +// QueueMockEmpty returns an empty Queue type. +func QueueMockEmpty() Queue { + qs := Queue{} + qs.SetRoutes([]string{}) + + return qs +} diff --git a/api/types/settings/queue_test.go b/api/types/settings/queue_test.go new file mode 100644 index 000000000..30389b881 --- /dev/null +++ b/api/types/settings/queue_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTypes_Queue_Getters(t *testing.T) { + // setup tests + tests := []struct { + queue *Queue + want *Queue + }{ + { + queue: testQueueSettings(), + want: testQueueSettings(), + }, + { + queue: new(Queue), + want: new(Queue), + }, + } + + // run tests + for _, test := range tests { + if !reflect.DeepEqual(test.queue.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("GetRoutes is %v, want %v", test.queue.GetRoutes(), test.want.GetRoutes()) + } + } +} + +func TestTypes_Queue_Setters(t *testing.T) { + // setup types + var qs *Queue + + // setup tests + tests := []struct { + queue *Queue + want *Queue + }{ + { + queue: testQueueSettings(), + want: testQueueSettings(), + }, + { + queue: qs, + want: new(Queue), + }, + } + + // run tests + for _, test := range tests { + test.queue.SetRoutes(test.want.GetRoutes()) + + if !reflect.DeepEqual(test.queue.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("SetRoutes is %v, want %v", test.queue.GetRoutes(), test.want.GetRoutes()) + } + } +} + +func TestTypes_Queue_String(t *testing.T) { + // setup types + qs := testQueueSettings() + + want := fmt.Sprintf(`{ + Routes: %s, +}`, + qs.GetRoutes(), + ) + + // run test + got := qs.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testQueueSettings is a test helper function to create a Queue +// type with all fields set to a fake value. +func testQueueSettings() *Queue { + qs := new(Queue) + + qs.SetRoutes([]string{"vela"}) + + return qs +} diff --git a/cmd/vela-server/compiler.go b/cmd/vela-server/compiler.go deleted file mode 100644 index 872f3fbd0..000000000 --- a/cmd/vela-server/compiler.go +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - - "github.com/go-vela/server/compiler" - "github.com/go-vela/server/compiler/native" - "github.com/go-vela/types/constants" -) - -// helper function to setup the queue from the CLI arguments. -func setupCompiler(c *cli.Context) (compiler.Engine, error) { - logrus.Debug("Creating queue client from CLI configuration") - return setupCompilerNative(c) -} - -// helper function to setup the Kafka queue from the CLI arguments. -func setupCompilerNative(c *cli.Context) (compiler.Engine, error) { - logrus.Tracef("Creating %s compiler client from CLI configuration", constants.DriverKafka) - return native.New(c) -} diff --git a/cmd/vela-server/main.go b/cmd/vela-server/main.go index 5ce4c7bd7..7fa51ca27 100644 --- a/cmd/vela-server/main.go +++ b/cmd/vela-server/main.go @@ -79,6 +79,12 @@ func main() { Name: "vela-secret", Usage: "secret used for server <-> agent communication", }, + &cli.DurationFlag{ + EnvVars: []string{"VELA_PLATFORM_SETTINGS_REFRESH_INTERVAL", "VELA_SETTINGS_REFRESH_INTERVAL"}, + Name: "settings-refresh-interval", + Usage: "interval at which platform settings will be refreshed", + Value: 5 * time.Second, + }, &cli.StringFlag{ EnvVars: []string{"VELA_SERVER_PRIVATE_KEY"}, Name: "vela-server-private-key", diff --git a/cmd/vela-server/queue.go b/cmd/vela-server/queue.go deleted file mode 100644 index 7a3a74c41..000000000 --- a/cmd/vela-server/queue.go +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" - - "github.com/go-vela/server/queue" -) - -// helper function to setup the queue from the CLI arguments. -func setupQueue(c *cli.Context) (queue.Service, error) { - logrus.Debug("Creating queue client from CLI configuration") - - // queue configuration - _setup := &queue.Setup{ - Driver: c.String("queue.driver"), - Address: c.String("queue.addr"), - Cluster: c.Bool("queue.cluster"), - Routes: c.StringSlice("queue.routes"), - Timeout: c.Duration("queue.pop.timeout"), - PrivateKey: c.String("queue.private-key"), - PublicKey: c.String("queue.public-key"), - } - - // setup the queue - // - // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New - return queue.New(_setup) -} diff --git a/cmd/vela-server/schedule.go b/cmd/vela-server/schedule.go index b86e70b79..f2832935e 100644 --- a/cmd/vela-server/schedule.go +++ b/cmd/vela-server/schedule.go @@ -14,6 +14,7 @@ import ( "github.com/go-vela/server/api/build" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" @@ -29,7 +30,7 @@ const ( scheduleWait = "waiting to trigger build for schedule" ) -func processSchedules(ctx context.Context, start time.Time, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedules(ctx context.Context, start time.Time, settings *settings.Platform, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service) error { logrus.Infof("processing active schedules to create builds") // send API call to capture the list of active schedules @@ -122,7 +123,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En } // process the schedule and trigger a new build - err = processSchedule(ctx, schedule, compiler, database, metadata, queue, scm, allowList) + err = processSchedule(ctx, schedule, settings, compiler, database, metadata, queue, scm) if err != nil { handleError(ctx, database, err, schedule) @@ -147,7 +148,7 @@ func processSchedules(ctx context.Context, start time.Time, compiler compiler.En } // processSchedule will, given a schedule, process it and trigger a new build. -func processSchedule(ctx context.Context, s *api.Schedule, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service, allowList []string) error { +func processSchedule(ctx context.Context, s *api.Schedule, settings *settings.Platform, compiler compiler.Engine, database database.Interface, metadata *internal.Metadata, queue queue.Service, scm scm.Service) error { // send API call to capture the repo for the schedule r, err := database.GetRepo(ctx, s.GetRepo().GetID()) if err != nil { @@ -155,7 +156,7 @@ func processSchedule(ctx context.Context, s *api.Schedule, compiler compiler.Eng } // ensure repo has not been removed from allow list - if !util.CheckAllowlist(r, allowList) { + if !util.CheckAllowlist(r, settings.GetScheduleAllowlist()) { return fmt.Errorf("skipping schedule: repo %s no longer on allow list", r.GetFullName()) } diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index 7f25a43df..bae9ec326 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -4,6 +4,7 @@ package main import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -16,13 +17,18 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" + "gorm.io/gorm" "k8s.io/apimachinery/pkg/util/wait" + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/compiler/native" "github.com/go-vela/server/database" + "github.com/go-vela/server/queue" "github.com/go-vela/server/router" "github.com/go-vela/server/router/middleware" ) +//nolint:funlen,gocyclo // ignore function length and cyclomatic complexity func server(c *cli.Context) error { // set log formatter switch c.String("log-formatter") { @@ -67,7 +73,7 @@ func server(c *cli.Context) error { logrus.SetLevel(logrus.PanicLevel) } - compiler, err := setupCompiler(c) + compiler, err := native.FromCLIContext(c) if err != nil { return err } @@ -77,7 +83,7 @@ func server(c *cli.Context) error { return err } - queue, err := setupQueue(c) + queue, err := queue.FromCLIContext(c) if err != nil { return err } @@ -97,7 +103,59 @@ func server(c *cli.Context) error { return err } + jitter := wait.Jitter(5*time.Second, 2.0) + + logrus.Infof("retrieving initial platform settings after %v delay", jitter) + + time.Sleep(jitter) + + ps, err := database.GetSettings(context.Background()) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + // platform settings record does not exist + if err != nil { + logrus.Info("creating initial platform settings") + + // create initial settings record + ps = new(settings.Platform) + + // singleton record ID should always be 1 + ps.SetID(1) + + ps.SetCreatedAt(time.Now().UTC().Unix()) + ps.SetUpdatedAt(time.Now().UTC().Unix()) + ps.SetUpdatedBy("vela-server") + + // read in defaults supplied from the cli runtime + compilerSettings := compiler.GetSettings() + ps.SetCompiler(compilerSettings) + + queueSettings := queue.GetSettings() + ps.SetQueue(queueSettings) + + // set repos permitted to be added + ps.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) + + // set repos permitted to use schedules + ps.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) + + // create the settings record in the database + _, err = database.CreateSettings(context.Background(), ps) + if err != nil { + return err + } + } + + // update any internal settings, this occurs in middleware + // to keep settings refreshed for each request + queue.SetSettings(ps) + compiler.SetSettings(ps) + router := router.Load( + middleware.CLI(c), + middleware.Settings(ps), middleware.Compiler(compiler), middleware.Database(database), middleware.Logger(logrus.StandardLogger(), time.RFC3339), @@ -111,7 +169,6 @@ func server(c *cli.Context) error { middleware.QueueSigningPrivateKey(c.String("queue.private-key")), middleware.QueueSigningPublicKey(c.String("queue.public-key")), middleware.QueueAddress(c.String("queue.addr")), - middleware.Allowlist(c.StringSlice("vela-repo-allowlist")), middleware.DefaultBuildLimit(c.Int64("default-build-limit")), middleware.DefaultTimeout(c.Int64("default-build-timeout")), middleware.MaxBuildLimit(c.Int64("max-build-limit")), @@ -121,7 +178,6 @@ func server(c *cli.Context) error { middleware.DefaultRepoEvents(c.StringSlice("default-repo-events")), middleware.DefaultRepoEventsMask(c.Int64("default-repo-events-mask")), middleware.DefaultRepoApproveBuild(c.String("default-repo-approve-build")), - middleware.AllowlistSchedule(c.StringSlice("vela-schedule-allowlist")), middleware.ScheduleFrequency(c.Duration("schedule-minimum-frequency")), ) @@ -158,26 +214,52 @@ func server(c *cli.Context) error { select { case sig := <-signalChannel: logrus.Infof("received signal: %s", sig) + err := srv.Shutdown(ctx) if err != nil { logrus.Error(err) } + done() case <-gctx.Done(): logrus.Info("closing signal goroutine") + err := srv.Shutdown(ctx) if err != nil { logrus.Error(err) } + return gctx.Err() } return nil }) + // spawn goroutine for refreshing settings + g.Go(func() error { + interval := c.Duration("settings-refresh-interval") + + logrus.Infof("refreshing platform settings every %v", interval) + + for { + time.Sleep(interval) + + newSettings, err := database.GetSettings(context.Background()) + if err != nil { + logrus.WithError(err).Warn("unable to refresh platform settings") + + continue + } + + // update the internal fields for the shared settings record + ps.Update(newSettings) + } + }) + // spawn goroutine for starting the server g.Go(func() error { logrus.Infof("starting server on %s", addr.Host) + err = srv.ListenAndServe() if err != nil { // log a message indicating the failure of the server @@ -190,6 +272,7 @@ func server(c *cli.Context) error { // spawn goroutine for starting the scheduler g.Go(func() error { logrus.Info("starting scheduler") + for { // track the starting time for when the server begins processing schedules // @@ -214,7 +297,11 @@ func server(c *cli.Context) error { // sleep for a duration of time before processing schedules time.Sleep(jitter) - err = processSchedules(ctx, start, compiler, database, metadata, queue, scm, c.StringSlice("vela-schedule-allowlist")) + // update internal settings updated through refresh + compiler.SetSettings(ps) + queue.SetSettings(ps) + + err = processSchedules(ctx, start, ps, compiler, database, metadata, queue, scm) if err != nil { logrus.WithError(err).Warn("unable to process schedules") } else { diff --git a/compiler/engine.go b/compiler/engine.go index 3079148d4..59dca1469 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -4,6 +4,7 @@ package compiler import ( api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/internal" "github.com/go-vela/types/library" "github.com/go-vela/types/pipeline" @@ -144,7 +145,13 @@ type Engine interface { // WithLabel defines a function that sets // the label(s) in the Engine. WithLabels([]string) Engine - // WithUser defines a function that sets + // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(string, string) Engine + // GetSettings defines a function that returns new api settings + // with the compiler Engine fields filled. + GetSettings() settings.Compiler + // SetSettings defines a function that takes api settings + // and updates the compiler Engine. + SetSettings(*settings.Platform) } diff --git a/compiler/native/clone.go b/compiler/native/clone.go index 9075292f7..85016cfc6 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -30,7 +30,7 @@ func (c *client) CloneStage(p *yaml.Build) (*yaml.Build, error) { Steps: yaml.StepSlice{ &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, @@ -63,7 +63,7 @@ func (c *client) CloneStep(p *yaml.Build) (*yaml.Build, error) { // create new clone step clone := &yaml.Step{ Detach: false, - Image: c.CloneImage, + Image: c.GetCloneImage(), Name: cloneStepName, Privileged: false, Pull: constants.PullNotPresent, diff --git a/compiler/native/clone_test.go b/compiler/native/clone_test.go index 120e3524f..304f703b2 100644 --- a/compiler/native/clone_test.go +++ b/compiler/native/clone_test.go @@ -84,7 +84,7 @@ func TestNative_CloneStage(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } @@ -167,7 +167,7 @@ func TestNative_CloneStep(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/compile.go b/compiler/native/compile.go index c33035149..0c739c310 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -75,7 +75,7 @@ func (c *client) Compile(v interface{}) (*pipeline.Build, *library.Pipeline, err switch { case p.Metadata.RenderInline: - newPipeline, err := c.compileInline(p, c.TemplateDepth) + newPipeline, err := c.compileInline(p, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -110,7 +110,7 @@ func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substit _pipeline.SetType(c.repo.GetPipelineType()) if p.Metadata.RenderInline { - newPipeline, err := c.compileInline(p, c.TemplateDepth) + newPipeline, err := c.compileInline(p, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -167,7 +167,7 @@ func (c *client) CompileLite(v interface{}, ruleData *pipeline.RuleData, substit case len(p.Steps) > 0: // inject the templates into the steps - p, err = c.ExpandSteps(p, templates, ruleData, c.TemplateDepth) + p, err = c.ExpandSteps(p, templates, ruleData, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } @@ -209,7 +209,7 @@ func (c *client) compileInline(p *yaml.Build, depth int) (*yaml.Build, error) { // return if max template depth has been reached if depth == 0 { - retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + retErr := fmt.Errorf("max template depth of %d exceeded", c.GetTemplateDepth()) return nil, retErr } @@ -318,7 +318,7 @@ func (c *client) compileSteps(p *yaml.Build, _pipeline *library.Pipeline, tmpls } // inject the templates into the steps - p, err = c.ExpandSteps(p, tmpls, r, c.TemplateDepth) + p, err = c.ExpandSteps(p, tmpls, r, c.GetTemplateDepth()) if err != nil { return nil, _pipeline, err } diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 811f9855e..15f2ab8f3 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -244,7 +244,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -582,7 +582,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -841,7 +841,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1086,7 +1086,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1207,7 +1207,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1328,7 +1328,7 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1370,6 +1370,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -1407,7 +1408,7 @@ func TestNative_Compile_InvalidType(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1595,7 +1596,7 @@ func TestNative_Compile_Clone(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1803,7 +1804,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -1828,6 +1829,7 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { func TestNative_Compile_NoStepsorStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) name := "foo" author := "author" @@ -1839,11 +1841,15 @@ func TestNative_Compile_NoStepsorStages(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } + // todo: this needs to be fixed in compiler validation + // this is a dirty hack to make this test pass + compiler.SetCloneImage("") + compiler.repo = &api.Repo{Name: &author} compiler.build = &api.Build{Author: &name, Number: &number} @@ -1860,6 +1866,7 @@ func TestNative_Compile_NoStepsorStages(t *testing.T) { func TestNative_Compile_StepsandStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) name := "foo" author := "author" @@ -1871,7 +1878,7 @@ func TestNative_Compile_StepsandStages(t *testing.T) { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -2952,7 +2959,7 @@ func Test_Compile_Inline(t *testing.T) { if err != nil { t.Errorf("Reading yaml file return err: %v", err) } - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -3016,6 +3023,7 @@ func Test_CompileLite(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -3823,7 +3831,7 @@ func Test_CompileLite(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index 1e090dad1..9c55ed41d 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -20,6 +20,7 @@ import ( func TestNative_EnvironmentStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -60,7 +61,7 @@ func TestNative_EnvironmentStages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -78,6 +79,7 @@ func TestNative_EnvironmentStages(t *testing.T) { func TestNative_EnvironmentSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ @@ -205,7 +207,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { } // run test non-local - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -253,6 +255,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { func TestNative_EnvironmentServices(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ @@ -380,7 +383,7 @@ func TestNative_EnvironmentServices(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -398,6 +401,7 @@ func TestNative_EnvironmentServices(t *testing.T) { func TestNative_EnvironmentSecrets(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) e := raw.StringSliceMap{ @@ -538,7 +542,7 @@ func TestNative_EnvironmentSecrets(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/expand.go b/compiler/native/expand.go index 335ca5007..365946d60 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -28,7 +28,7 @@ func (c *client) ExpandStages(s *yaml.Build, tmpls map[string]*yaml.Template, r // iterate through all stages for _, stage := range s.Stages { // inject the templates into the steps for the stage - p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.TemplateDepth) + p, err := c.ExpandSteps(&yaml.Build{Steps: stage.Steps, Secrets: s.Secrets, Services: s.Services, Environment: s.Environment}, tmpls, r, c.GetTemplateDepth()) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (c *client) ExpandSteps(s *yaml.Build, tmpls map[string]*yaml.Template, r * // return if max template depth has been reached if depth == 0 { - retErr := fmt.Errorf("max template depth of %d exceeded", c.TemplateDepth) + retErr := fmt.Errorf("max template depth of %d exceeded", c.GetTemplateDepth()) return s, retErr } @@ -349,7 +349,7 @@ func (c *client) mergeTemplate(bytes []byte, tmpl *yaml.Template, step *yaml.Ste return native.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables) case constants.PipelineTypeStarlark: //nolint:lll // ignore long line length due to return - return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables, c.StarlarkExecLimit) + return starlark.Render(string(bytes), step.Name, step.Template.Name, step.Environment, step.Template.Variables, c.GetStarlarkExecLimit()) default: //nolint:lll // ignore long line length due to return return &yaml.Build{}, fmt.Errorf("format of %s is unsupported", tmpl.Format) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 0e69eed65..2f036f974 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -44,6 +44,7 @@ func TestNative_ExpandStages(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -144,7 +145,7 @@ func TestNative_ExpandStages(t *testing.T) { } // run test -- missing private github - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -165,7 +166,7 @@ func TestNative_ExpandStages(t *testing.T) { } // run test - compiler, err = New(c) + compiler, err = FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -226,6 +227,7 @@ func TestNative_ExpandSteps(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testRepo := new(api.Repo) @@ -346,7 +348,7 @@ func TestNative_ExpandSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -355,7 +357,7 @@ func TestNative_ExpandSteps(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -404,6 +406,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -615,7 +618,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { wantEnvironment := raw.StringSliceMap{} // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -623,7 +626,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { ruledata := new(pipeline.RuleData) ruledata.Branch = "main" - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, ruledata, compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -670,6 +673,7 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tmpls := map[string]*yaml.Template{ @@ -708,12 +712,12 @@ func TestNative_ExpandStepsStarlark(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Secrets: yaml.SecretSlice{}, Services: yaml.ServiceSlice{}, Environment: raw.StringSliceMap{}}, tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps returned err: %v", err) } @@ -760,6 +764,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) @@ -883,7 +888,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -892,7 +897,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment, Templates: yaml.TemplateSlice{test.tmpls["chain"]}}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + build, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment, Templates: yaml.TemplateSlice{test.tmpls["chain"]}}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err != nil { t.Errorf("ExpandSteps_Type%s returned err: %v", test.name, err) } @@ -945,6 +950,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) @@ -989,7 +995,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -998,7 +1004,7 @@ func TestNative_ExpandSteps_TemplateCallTemplate_CircularFail(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err == nil { t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) } @@ -1031,6 +1037,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { set.String("github-url", s.URL, "doc") set.String("github-token", "", "doc") set.Int("max-template-depth", 5, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) testBuild := new(api.Build) @@ -1075,7 +1082,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating new compiler returned err: %v", err) } @@ -1084,7 +1091,7 @@ func TestNative_ExpandSteps_CallTemplateWithRenderInline(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.TemplateDepth) + _, err := compiler.ExpandSteps(&yaml.Build{Steps: steps, Services: yaml.ServiceSlice{}, Environment: globalEnvironment}, test.tmpls, new(pipeline.RuleData), compiler.GetTemplateDepth()) if err == nil { t.Errorf("ExpandSteps_Type%s should have returned an error", test.name) } diff --git a/compiler/native/initialize_test.go b/compiler/native/initialize_test.go index 33276ffdb..ea196216a 100644 --- a/compiler/native/initialize_test.go +++ b/compiler/native/initialize_test.go @@ -15,6 +15,7 @@ import ( func TestNative_InitStage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -61,7 +62,7 @@ func TestNative_InitStage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -79,6 +80,7 @@ func TestNative_InitStage(t *testing.T) { func TestNative_InitStep(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -109,7 +111,7 @@ func TestNative_InitStep(t *testing.T) { }, } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/native.go b/compiler/native/native.go index 026341063..93d93c5ea 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -3,16 +3,19 @@ package native import ( + "fmt" "time" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" "github.com/go-vela/server/internal" + "github.com/go-vela/server/internal/image" ) type ModificationConfig struct { @@ -27,9 +30,8 @@ type client struct { PrivateGithub registry.Service UsePrivateGithub bool ModificationService ModificationConfig - CloneImage string - TemplateDepth int - StarlarkExecLimit uint64 + + settings.Compiler build *api.Build comment string @@ -43,10 +45,10 @@ type client struct { labels []string } -// New returns a Pipeline implementation that integrates with the supported registries. +// FromCLIContext returns a Pipeline implementation that integrates with the supported registries. // //nolint:revive // ignore returning unexported client -func New(ctx *cli.Context) (*client, error) { +func FromCLIContext(ctx *cli.Context) (*client, error) { logrus.Debug("Creating registry clients from CLI configuration") c := new(client) @@ -68,14 +70,24 @@ func New(ctx *cli.Context) (*client, error) { c.Github = github + c.Compiler = settings.Compiler{} + + cloneImage := ctx.String("clone-image") + + // validate clone image + _, err = image.ParseWithError(cloneImage) + if err != nil { + return nil, fmt.Errorf("invalid clone image %s: %w", cloneImage, err) + } + // set the clone image to use for the injected clone step - c.CloneImage = ctx.String("clone-image") + c.SetCloneImage(cloneImage) // set the template depth to use for nested templates - c.TemplateDepth = ctx.Int("max-template-depth") + c.SetTemplateDepth(ctx.Int("max-template-depth")) // set the starlark execution step limit for compiling starlark pipelines - c.StarlarkExecLimit = ctx.Uint64("compiler-starlark-exec-limit") + c.SetStarlarkExecLimit(ctx.Uint64("compiler-starlark-exec-limit")) if ctx.Bool("github-driver") { logrus.Tracef("setting up Private GitHub Client for %s", ctx.String("github-url")) diff --git a/compiler/native/native_test.go b/compiler/native/native_test.go index 835169e5d..6c5b6d2be 100644 --- a/compiler/native/native_test.go +++ b/compiler/native/native_test.go @@ -10,6 +10,7 @@ import ( "github.com/urfave/cli/v2" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler/registry/github" "github.com/go-vela/server/internal" ) @@ -17,14 +18,17 @@ import ( func TestNative_New(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") want := &client{ - Github: public, + Github: public, + Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -43,6 +47,7 @@ func TestNative_New_PrivateGithub(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") private, _ := github.New(url, token) @@ -50,10 +55,12 @@ func TestNative_New_PrivateGithub(t *testing.T) { Github: public, PrivateGithub: private, UsePrivateGithub: true, + Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -72,6 +79,7 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) public, _ := github.New("", "") private, _ := github.New(url, token) @@ -79,10 +87,12 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { Github: public, PrivateGithub: private, UsePrivateGithub: true, + Compiler: settings.CompilerMockEmpty(), } + want.SetCloneImage(defaultCloneImage) // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("New returned err: %v", err) @@ -96,15 +106,16 @@ func TestNative_DuplicateRetainSettings(t *testing.T) { func TestNative_DuplicateStripBuild(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) b := &api.Build{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -119,16 +130,17 @@ func TestNative_DuplicateStripBuild(t *testing.T) { func TestNative_WithBuild(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) b := &api.Build{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.build = b // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -141,15 +153,16 @@ func TestNative_WithBuild(t *testing.T) { func TestNative_WithFiles(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) f := []string{"foo"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.files = f // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -162,14 +175,15 @@ func TestNative_WithFiles(t *testing.T) { func TestNative_WithComment(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) comment := "ok to test" - want, _ := New(c) + want, _ := FromCLIContext(c) want.comment = comment // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -182,14 +196,15 @@ func TestNative_WithComment(t *testing.T) { func TestNative_WithLocal(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) local := true - want, _ := New(c) + want, _ := FromCLIContext(c) want.local = true // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -202,14 +217,15 @@ func TestNative_WithLocal(t *testing.T) { func TestNative_WithLocalTemplates(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) localTemplates := []string{"example:tmpl.yml", "exmpl:template.yml"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.localTemplates = []string{"example:tmpl.yml", "exmpl:template.yml"} // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -222,6 +238,7 @@ func TestNative_WithLocalTemplates(t *testing.T) { func TestNative_WithMetadata(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -244,11 +261,11 @@ func TestNative_WithMetadata(t *testing.T) { }, } - want, _ := New(c) + want, _ := FromCLIContext(c) want.metadata = m // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -266,15 +283,16 @@ func TestNative_WithPrivateGitHub(t *testing.T) { set.Bool("github-driver", true, "doc") set.String("github-url", url, "doc") set.String("github-token", token, "doc") + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) private, _ := github.New(url, token) - want, _ := New(c) + want, _ := FromCLIContext(c) want.PrivateGithub = private // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -287,16 +305,17 @@ func TestNative_WithPrivateGitHub(t *testing.T) { func TestNative_WithRepo(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) r := &api.Repo{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.repo = r // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -309,16 +328,17 @@ func TestNative_WithRepo(t *testing.T) { func TestNative_WithUser(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) id := int64(1) u := &api.User{ID: &id} - want, _ := New(c) + want, _ := FromCLIContext(c) want.user = u // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -331,14 +351,15 @@ func TestNative_WithUser(t *testing.T) { func TestNative_WithLabels(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) labels := []string{"documentation", "enhancement"} - want, _ := New(c) + want, _ := FromCLIContext(c) want.labels = []string{"documentation", "enhancement"} // run test - got, err := New(c) + got, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/compiler/native/parse.go b/compiler/native/parse.go index 9ccd525f1..0cd7a5819 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -71,7 +71,7 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ // capture the raw pipeline configuration raw = []byte(parsedRaw) - p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables, c.StarlarkExecLimit) + p, err = starlark.RenderBuild(template.Name, parsedRaw, c.EnvironmentBuild(), template.Variables, c.GetStarlarkExecLimit()) if err != nil { return nil, raw, err } diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index dfe691fdb..e0aa9c512 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -21,7 +21,7 @@ import ( func TestNative_Parse_Metadata_Bytes(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -49,7 +49,7 @@ func TestNative_Parse_Metadata_Bytes(t *testing.T) { func TestNative_Parse_Metadata_File(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -79,7 +79,7 @@ func TestNative_Parse_Metadata_File(t *testing.T) { func TestNative_Parse_Metadata_Invalid(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) // run test got, _, err := client.Parse(nil, "", new(yaml.Template)) @@ -95,7 +95,7 @@ func TestNative_Parse_Metadata_Invalid(t *testing.T) { func TestNative_Parse_Metadata_Path(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -118,7 +118,7 @@ func TestNative_Parse_Metadata_Path(t *testing.T) { func TestNative_Parse_Metadata_Reader(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -146,7 +146,7 @@ func TestNative_Parse_Metadata_Reader(t *testing.T) { func TestNative_Parse_Metadata_String(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -174,7 +174,7 @@ func TestNative_Parse_Metadata_String(t *testing.T) { func TestNative_Parse_Parameters(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -221,7 +221,7 @@ func TestNative_Parse_Parameters(t *testing.T) { func TestNative_Parse_StagesPipeline(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -350,7 +350,7 @@ func TestNative_Parse_StagesPipeline(t *testing.T) { func TestNative_Parse_StepsPipeline(t *testing.T) { // setup types tBool := true - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Version: "1", Metadata: yaml.Metadata{ @@ -452,7 +452,7 @@ func TestNative_Parse_StepsPipeline(t *testing.T) { func TestNative_Parse_Secrets(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -522,7 +522,7 @@ func TestNative_Parse_Secrets(t *testing.T) { func TestNative_Parse_Stages(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, @@ -598,7 +598,7 @@ func TestNative_Parse_Stages(t *testing.T) { func TestNative_Parse_Steps(t *testing.T) { // setup types - client, _ := New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) want := &yaml.Build{ Metadata: yaml.Metadata{ Environment: []string{"steps", "services", "secrets"}, diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 308023e77..0f03e6865 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -16,6 +16,7 @@ import ( func TestNative_ScriptStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) baseEnv := environment(nil, nil, nil, nil) @@ -87,7 +88,7 @@ func TestNative_ScriptStages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -105,6 +106,7 @@ func TestNative_ScriptStages(t *testing.T) { func TestNative_ScriptSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) emptyEnv := environment(nil, nil, nil, nil) @@ -313,7 +315,7 @@ func TestNative_ScriptSteps(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/settings.go b/compiler/native/settings.go new file mode 100644 index 000000000..12997ec9c --- /dev/null +++ b/compiler/native/settings.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 + +package native + +import ( + "github.com/go-vela/server/api/types/settings" +) + +// GetSettings retrieves the api settings type. +func (c *client) GetSettings() settings.Compiler { + return c.Compiler +} + +// SetSettings sets the api settings type. +func (c *client) SetSettings(s *settings.Platform) { + if s != nil { + c.SetCloneImage(s.GetCloneImage()) + c.SetTemplateDepth(s.GetTemplateDepth()) + c.SetStarlarkExecLimit(s.GetStarlarkExecLimit()) + } +} diff --git a/compiler/native/substitute_test.go b/compiler/native/substitute_test.go index 459bb2896..0be8eac03 100644 --- a/compiler/native/substitute_test.go +++ b/compiler/native/substitute_test.go @@ -19,6 +19,7 @@ func Test_client_SubstituteStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tests := []struct { @@ -112,7 +113,7 @@ func Test_client_SubstituteStages(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } @@ -137,6 +138,7 @@ func Test_client_SubstituteSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) tests := []struct { @@ -236,7 +238,7 @@ func Test_client_SubstituteSteps(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Creating compiler returned err: %v", err) } diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index e8d596be3..bcd318103 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -17,6 +17,7 @@ import ( func TestNative_TransformStages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -220,7 +221,7 @@ func TestNative_TransformStages(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } @@ -257,6 +258,7 @@ func TestNative_TransformStages(t *testing.T) { func TestNative_TransformSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) m := &internal.Metadata{ @@ -439,7 +441,7 @@ func TestNative_TransformSteps(t *testing.T) { // run tests for _, test := range tests { - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("unable to create new compiler: %v", err) } diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 85896b4a1..226c75aa9 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -16,12 +16,13 @@ import ( func TestNative_Validate_NoVersion(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{} // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -35,6 +36,7 @@ func TestNative_Validate_NoVersion(t *testing.T) { func TestNative_Validate_NoStagesOrSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{ @@ -42,7 +44,7 @@ func TestNative_Validate_NoStagesOrSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -56,6 +58,7 @@ func TestNative_Validate_NoStagesOrSteps(t *testing.T) { func TestNative_Validate_StagesAndSteps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -85,7 +88,7 @@ func TestNative_Validate_StagesAndSteps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -99,6 +102,7 @@ func TestNative_Validate_StagesAndSteps(t *testing.T) { func TestNative_Validate_Services(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -122,7 +126,7 @@ func TestNative_Validate_Services(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -136,6 +140,7 @@ func TestNative_Validate_Services(t *testing.T) { func TestNative_Validate_Services_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -159,7 +164,7 @@ func TestNative_Validate_Services_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -173,6 +178,7 @@ func TestNative_Validate_Services_NoName(t *testing.T) { func TestNative_Validate_Services_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -196,7 +202,7 @@ func TestNative_Validate_Services_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -210,6 +216,7 @@ func TestNative_Validate_Services_NoImage(t *testing.T) { func TestNative_Validate_Stages(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -231,7 +238,7 @@ func TestNative_Validate_Stages(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -245,6 +252,7 @@ func TestNative_Validate_Stages(t *testing.T) { func TestNative_Validate_Stages_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -265,7 +273,7 @@ func TestNative_Validate_Stages_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -279,6 +287,7 @@ func TestNative_Validate_Stages_NoName(t *testing.T) { func TestNative_Validate_Stages_NoStepName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -299,7 +308,7 @@ func TestNative_Validate_Stages_NoStepName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -313,6 +322,7 @@ func TestNative_Validate_Stages_NoStepName(t *testing.T) { func TestNative_Validate_Stages_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -333,7 +343,7 @@ func TestNative_Validate_Stages_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -347,6 +357,7 @@ func TestNative_Validate_Stages_NoImage(t *testing.T) { func TestNative_Validate_Stages_NoCommands(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -367,7 +378,7 @@ func TestNative_Validate_Stages_NoCommands(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -381,6 +392,7 @@ func TestNative_Validate_Stages_NoCommands(t *testing.T) { func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -403,7 +415,7 @@ func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -417,6 +429,7 @@ func TestNative_Validate_Stages_NeedsSelfReference(t *testing.T) { func TestNative_Validate_Steps(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -433,7 +446,7 @@ func TestNative_Validate_Steps(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -447,6 +460,7 @@ func TestNative_Validate_Steps(t *testing.T) { func TestNative_Validate_Steps_NoName(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) p := &yaml.Build{ @@ -461,7 +475,7 @@ func TestNative_Validate_Steps_NoName(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -475,6 +489,7 @@ func TestNative_Validate_Steps_NoName(t *testing.T) { func TestNative_Validate_Steps_NoImage(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -490,7 +505,7 @@ func TestNative_Validate_Steps_NoImage(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -504,6 +519,7 @@ func TestNative_Validate_Steps_NoImage(t *testing.T) { func TestNative_Validate_Steps_NoCommands(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -519,7 +535,7 @@ func TestNative_Validate_Steps_NoCommands(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -533,6 +549,7 @@ func TestNative_Validate_Steps_NoCommands(t *testing.T) { func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -556,7 +573,7 @@ func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } @@ -571,6 +588,7 @@ func TestNative_Validate_Steps_ExceedReportAs(t *testing.T) { func TestNative_Validate_MultiReportAs(t *testing.T) { // setup types set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) str := "foo" @@ -595,7 +613,7 @@ func TestNative_Validate_MultiReportAs(t *testing.T) { } // run test - compiler, err := New(c) + compiler, err := FromCLIContext(c) if err != nil { t.Errorf("Unable to create new compiler: %v", err) } diff --git a/database/database.go b/database/database.go index 50d8ae956..5a9e982b2 100644 --- a/database/database.go +++ b/database/database.go @@ -23,6 +23,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -61,6 +62,7 @@ type ( // sirupsen/logrus logger used in database functions logger *logrus.Entry + settings.SettingsInterface build.BuildInterface dashboard.DashboardInterface executable.BuildExecutableInterface diff --git a/database/integration_test.go b/database/integration_test.go index 4f28bbb4e..db70091b3 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -13,6 +13,7 @@ import ( "github.com/google/go-cmp/cmp" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/dashboard" "github.com/go-vela/server/database/deployment" @@ -24,6 +25,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + dbSettings "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/testutils" "github.com/go-vela/server/database/user" @@ -49,6 +51,7 @@ type Resources struct { Steps []*library.Step Users []*api.User Workers []*api.Worker + Platform []*settings.Platform } func TestDatabase_Integration(t *testing.T) { @@ -150,6 +153,8 @@ func TestDatabase_Integration(t *testing.T) { t.Run("test_workers", func(t *testing.T) { testWorkers(t, db, resources) }) + t.Run("test_settings", func(t *testing.T) { testSettings(t, db, resources) }) + err = db.Close() if err != nil { t.Errorf("unable to close database engine for %s: %v", test.name, err) @@ -2165,6 +2170,56 @@ func testWorkers(t *testing.T, db Interface, resources *Resources) { } } +func testSettings(t *testing.T, db Interface, resources *Resources) { + // create a variable to track the number of methods called for settings + methods := make(map[string]bool) + // capture the element type of the settings interface + element := reflect.TypeOf(new(dbSettings.SettingsInterface)).Elem() + // iterate through all methods found in the settings interface + for i := 0; i < element.NumMethod(); i++ { + // skip tracking the methods to create indexes and tables for settings + // since those are already called when the database engine starts + if strings.Contains(element.Method(i).Name, "Index") || + strings.Contains(element.Method(i).Name, "Table") { + continue + } + + // add the method name to the list of functions + methods[element.Method(i).Name] = false + } + + // create the settings + for _, s := range resources.Platform { + _, err := db.CreateSettings(context.TODO(), s) + if err != nil { + t.Errorf("unable to create settings %d: %v", s.GetID(), err) + } + } + methods["CreateSettings"] = true + + // update the settings + for _, s := range resources.Platform { + s.SetCloneImage("target/vela-git:abc123") + got, err := db.UpdateSettings(context.TODO(), s) + if err != nil { + t.Errorf("unable to update settings %d: %v", s.GetID(), err) + } + + if !cmp.Equal(got, s) { + t.Errorf("UpdateSettings() is %v, want %v", got, s) + } + } + methods["UpdateSettings"] = true + methods["GetSettings"] = true + + // ensure we called all the methods we expected to + for method, called := range methods { + if !called { + t.Errorf("method %s was not called for settings", method) + } + } +} + func newResources() *Resources { userOne := new(api.User) userOne.SetID(1) diff --git a/database/interface.go b/database/interface.go index 16a0fdf27..0d82da099 100644 --- a/database/interface.go +++ b/database/interface.go @@ -14,6 +14,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -34,6 +35,9 @@ type Interface interface { // Resource Interface Functions + // SettingsInterface defines the interface for platform settings stored in the database. + settings.SettingsInterface + // BuildInterface defines the interface for builds stored in the database. build.BuildInterface diff --git a/database/resource.go b/database/resource.go index 21b805227..486d2a137 100644 --- a/database/resource.go +++ b/database/resource.go @@ -16,15 +16,29 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" ) // NewResources creates and returns the database agnostic engines for resources. +// +//nolint:funlen // ignore function length func (e *engine) NewResources(ctx context.Context) error { var err error + // create the database agnostic engine for settings + e.SettingsInterface, err = settings.New( + settings.WithContext(e.ctx), + settings.WithClient(e.client), + settings.WithLogger(e.logger), + settings.WithSkipCreation(e.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic engine for builds e.BuildInterface, err = build.New( build.WithContext(e.ctx), diff --git a/database/resource_test.go b/database/resource_test.go index dc99bb63b..a1371108b 100644 --- a/database/resource_test.go +++ b/database/resource_test.go @@ -19,6 +19,7 @@ import ( "github.com/go-vela/server/database/schedule" "github.com/go-vela/server/database/secret" "github.com/go-vela/server/database/service" + "github.com/go-vela/server/database/settings" "github.com/go-vela/server/database/step" "github.com/go-vela/server/database/user" "github.com/go-vela/server/database/worker" @@ -28,6 +29,8 @@ func TestDatabase_Engine_NewResources(t *testing.T) { _postgres, _mock := testPostgres(t) defer _postgres.Close() + // ensure the mock expects the settings queries + _mock.ExpectExec(settings.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the build queries _mock.ExpectExec(build.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(build.CreateCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/database/secret/create_test.go b/database/secret/create_test.go index 7ec36795b..ed3d95591 100644 --- a/database/secret/create_test.go +++ b/database/secret/create_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/testutils" "github.com/go-vela/types/library" ) @@ -63,21 +64,21 @@ func TestSecret_Engine_CreateSecret(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). + WithArgs("foo", "bar", nil, "baz", testutils.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", 1, "user2", 1). WillReturnRows(_rows) // ensure the mock expects the org secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). + WithArgs("foo", "*", nil, "bar", testutils.AnyArgument{}, "org", nil, 3, false, false, 1, "user", 1, "user2", 2). WillReturnRows(_rows) // ensure the mock expects the shared secrets query _mock.ExpectQuery(`INSERT INTO "secrets" ("org","repo","team","name","value","type","images","allow_events","allow_command","allow_substitution","created_at","created_by","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING "id"`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). + WithArgs("foo", nil, "bar", "baz", testutils.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", 1, "user2", 3). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/secret/secret_test.go b/database/secret/secret_test.go index fdc10818c..41bb4dd3a 100644 --- a/database/secret/secret_test.go +++ b/database/secret/secret_test.go @@ -3,10 +3,8 @@ package secret import ( - "database/sql/driver" "reflect" "testing" - "time" "github.com/DATA-DOG/go-sqlmock" "github.com/sirupsen/logrus" @@ -251,29 +249,3 @@ func testEvents() *library.Events { }, } } - -// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values -// that are otherwise not easily compared. These typically would be values generated -// before adding or updating them in the database. -// -// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime -type AnyArgument struct{} - -// Match satisfies sqlmock.Argument interface. -func (a AnyArgument) Match(_ driver.Value) bool { - return true -} - -// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. -type NowTimestamp struct{} - -// Match satisfies sqlmock.Argument interface. -func (t NowTimestamp) Match(v driver.Value) bool { - ts, ok := v.(int64) - if !ok { - return false - } - now := time.Now().Unix() - - return now-ts < 10 -} diff --git a/database/secret/update_test.go b/database/secret/update_test.go index 11b6245a1..e53d43b4e 100644 --- a/database/secret/update_test.go +++ b/database/secret/update_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/server/database/testutils" "github.com/go-vela/types/library" ) @@ -60,21 +61,21 @@ func TestSecret_Engine_UpdateSecret(t *testing.T) { _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "bar", nil, "baz", AnyArgument{}, "repo", nil, 1, false, false, 1, "user", AnyArgument{}, "user2", 1). + WithArgs("foo", "bar", nil, "baz", testutils.AnyArgument{}, "repo", nil, 1, false, false, 1, "user", testutils.AnyArgument{}, "user2", 1). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the org query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", "*", nil, "bar", AnyArgument{}, "org", nil, 1, false, false, 1, "user", AnyArgument{}, "user2", 2). + WithArgs("foo", "*", nil, "bar", testutils.AnyArgument{}, "org", nil, 1, false, false, 1, "user", testutils.AnyArgument{}, "user2", 2). WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the shared query _mock.ExpectExec(`UPDATE "secrets" SET "org"=$1,"repo"=$2,"team"=$3,"name"=$4,"value"=$5,"type"=$6,"images"=$7,"allow_events"=$8,"allow_command"=$9,"allow_substitution"=$10,"created_at"=$11,"created_by"=$12,"updated_at"=$13,"updated_by"=$14 WHERE "id" = $15`). - WithArgs("foo", nil, "bar", "baz", AnyArgument{}, "shared", nil, 1, false, false, 1, "user", NowTimestamp{}, "user2", 3). + WithArgs("foo", nil, "bar", "baz", testutils.AnyArgument{}, "shared", nil, 1, false, false, 1, "user", testutils.NowTimestamp{}, "user2", 3). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/settings/create.go b/database/settings/create.go new file mode 100644 index 000000000..0e86f7fbc --- /dev/null +++ b/database/settings/create.go @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" +) + +// CreateSettings creates a platform settings record in the database. +func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { + e.logger.Tracef("creating platform settings in the database with %v", s.String()) + + // cast the api type to database type + settings := types.FromAPI(s) + + // validate the necessary fields are populated + err := settings.Validate() + if err != nil { + return nil, err + } + + // send query to the database + err = e.client.Table(TableSettings).Create(settings.Nullify()).Error + if err != nil { + return nil, err + } + + return s, nil +} diff --git a/database/settings/create_test.go b/database/settings/create_test.go new file mode 100644 index 000000000..e621bf76e --- /dev/null +++ b/database/settings/create_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSettings_Engine_CreateSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(1) + _settings.SetUpdatedBy("") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the query + _mock.ExpectQuery(`INSERT INTO "settings" ("compiler","queue","repo_allowlist","schedule_allowlist","created_at","updated_at","updated_by","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING "id"`). + WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, ``, 1). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CreateSettings(context.TODO(), _settings) + + if test.failure { + if err == nil { + t.Errorf("CreateSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, _settings) { + t.Errorf("CreateSettings for %s returned %s, want %s", test.name, got, _settings) + } + }) + } +} diff --git a/database/settings/get.go b/database/settings/get.go new file mode 100644 index 000000000..0bba2e0f4 --- /dev/null +++ b/database/settings/get.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" +) + +// GetSettings gets platform settings from the database. +func (e *engine) GetSettings(ctx context.Context) (*settings.Platform, error) { + e.logger.Trace("getting platform settings from the database") + + // variable to store query results + s := new(types.Platform) + + // send query to the database and store result in variable + err := e.client. + Table(TableSettings). + Where("id = ?", 1). + Take(s). + Error + if err != nil { + return nil, err + } + + // return the settings + return s.ToAPI(), nil +} diff --git a/database/settings/get_test.go b/database/settings/get_test.go new file mode 100644 index 000000000..83196848b --- /dev/null +++ b/database/settings/get_test.go @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_Engine_GetSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(1) + _settings.SetUpdatedBy("octocat") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "compiler", "queue", "repo_allowlist", "schedule_allowlist", "created_at", "updated_at", "updated_by"}). + AddRow(1, `{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, 1, `octocat`) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "settings" WHERE id = $1 LIMIT $2`).WithArgs(1, 1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateSettings(context.TODO(), _settings) + if err != nil { + t.Errorf("unable to create test settings for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *settings.Platform + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _settings, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _settings, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetSettings(context.TODO()) + + if test.failure { + if err == nil { + t.Errorf("GetSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetSettings for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/settings/interface.go b/database/settings/interface.go new file mode 100644 index 000000000..a7cc755cd --- /dev/null +++ b/database/settings/interface.go @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" +) + +// SettingsInterface represents the Vela interface for settings +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type SettingsInterface interface { + // CreateSettings defines a function that creates a platform settings record. + CreateSettings(context.Context, *settings.Platform) (*settings.Platform, error) + // GetSettings defines a function that gets platform settings. + GetSettings(context.Context) (*settings.Platform, error) + // UpdateSettings defines a function that updates platform settings. + UpdateSettings(context.Context, *settings.Platform) (*settings.Platform, error) +} diff --git a/database/settings/opts.go b/database/settings/opts.go new file mode 100644 index 000000000..a9646d1bf --- /dev/null +++ b/database/settings/opts.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Settings. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Settings. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the settings engine + e.client = client + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Settings. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the settings engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Settings. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the settings engine + e.config.SkipCreation = skipCreation + + return nil + } +} + +// WithContext sets the context in the database engine for Settings. +func WithContext(ctx context.Context) EngineOpt { + return func(e *engine) error { + e.ctx = ctx + + return nil + } +} diff --git a/database/settings/opts_test.go b/database/settings/opts_test.go new file mode 100644 index 000000000..6fccdec17 --- /dev/null +++ b/database/settings/opts_test.go @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +func TestSettings_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} + +func TestSettings_EngineOpt_WithContext(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + ctx context.Context + want context.Context + }{ + { + failure: false, + name: "context set to TODO", + ctx: context.TODO(), + want: context.TODO(), + }, + { + failure: false, + name: "context set to nil", + ctx: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithContext(test.ctx)(e) + + if test.failure { + if err == nil { + t.Errorf("WithContext for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithContext returned err: %v", err) + } + + if !reflect.DeepEqual(e.ctx, test.want) { + t.Errorf("WithContext is %v, want %v", e.ctx, test.want) + } + }) + } +} diff --git a/database/settings/settings.go b/database/settings/settings.go new file mode 100644 index 000000000..a673ed49d --- /dev/null +++ b/database/settings/settings.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "fmt" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" +) + +const ( + TableSettings = "settings" +) + +type ( + // config represents the settings required to create the engine that implements the SettingsInterface interface. + config struct { + // specifies to skip creating tables and indexes for the Settings engine + SkipCreation bool + } + + // engine represents the settings functionality that implements the SettingsInterface interface. + engine struct { + // engine configuration settings used in settings functions + config *config + + ctx context.Context + + // gorm.io/gorm database client used in settings functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in settings functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with settings in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Settings engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of settings table and indexes in the database") + + return e, nil + } + + // create the settings table + err := e.CreateSettingsTable(e.ctx, e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", TableSettings, err) + } + + return e, nil +} diff --git a/database/settings/settings_test.go b/database/settings/settings_test.go new file mode 100644 index 000000000..0e1ac773f --- /dev/null +++ b/database/settings/settings_test.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/sirupsen/logrus" + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres settings engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite settings engine: %v", err) + } + + return _engine +} + +// testSettings is a test helper function to create an api +// Platform type with all fields set to their zero values. +func testSettings() *settings.Platform { + s := settings.PlatformMockEmpty() + + return &s +} diff --git a/database/settings/table.go b/database/settings/table.go new file mode 100644 index 000000000..a7268bc76 --- /dev/null +++ b/database/settings/table.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres settings table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +settings ( + id SERIAL PRIMARY KEY, + compiler JSON DEFAULT NULL, + queue JSON DEFAULT NULL, + repo_allowlist VARCHAR(1000), + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by VARCHAR(250) +); +` + + // CreateSqliteTable represents a query to create the Sqlite settings table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +settings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + compiler TEXT, + queue TEXT, + repo_allowlist VARCHAR(1000), + schedule_allowlist VARCHAR(1000), + created_at INTEGER, + updated_at INTEGER, + updated_by TEXT +); +` +) + +// CreateSettingsTable creates the settings table in the database. +func (e *engine) CreateSettingsTable(_ context.Context, driver string) error { + e.logger.Tracef("creating settings table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the steps table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the steps table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/settings/table_test.go b/database/settings/table_test.go new file mode 100644 index 000000000..13eaab42b --- /dev/null +++ b/database/settings/table_test.go @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestSettings_Engine_CreateSettingsTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateSettingsTable(context.TODO(), test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateSettingsTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateSettingsTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/settings/update.go b/database/settings/update.go new file mode 100644 index 000000000..b72b4d543 --- /dev/null +++ b/database/settings/update.go @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/database/types" +) + +// UpdateSettings updates a platform settings in the database. +func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*settings.Platform, error) { + e.logger.Trace("updating platform settings in the database") + + // cast the api type to database type + dbS := types.FromAPI(s) + + // validate the necessary fields are populated + err := dbS.Validate() + if err != nil { + return nil, err + } + + // send query to the database + err = e.client.Table(TableSettings).Save(dbS.Nullify()).Error + if err != nil { + return nil, err + } + + s = dbS.ToAPI() + + return s, nil +} diff --git a/database/settings/update_test.go b/database/settings/update_test.go new file mode 100644 index 000000000..7b9f31a38 --- /dev/null +++ b/database/settings/update_test.go @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/go-vela/server/database/testutils" +) + +func TestSettings_Engine_UpdateSettings(t *testing.T) { + // setup types + _settings := testSettings() + _settings.SetID(1) + _settings.SetCloneImage("target/vela-git:latest") + _settings.SetTemplateDepth(10) + _settings.SetStarlarkExecLimit(100) + _settings.SetRoutes([]string{"vela", "large"}) + _settings.SetRepoAllowlist([]string{"octocat/hello-world"}) + _settings.SetScheduleAllowlist([]string{"*"}) + _settings.SetCreatedAt(1) + _settings.SetUpdatedAt(1) + _settings.SetUpdatedBy("octocat") + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`UPDATE "settings" SET "compiler"=$1,"queue"=$2,"repo_allowlist"=$3,"schedule_allowlist"=$4,"created_at"=$5,"updated_at"=$6,"updated_by"=$7 WHERE "id" = $8`). + WithArgs(`{"clone_image":{"String":"target/vela-git:latest","Valid":true},"template_depth":{"Int64":10,"Valid":true},"starlark_exec_limit":{"Int64":100,"Valid":true}}`, + `{"routes":["vela","large"]}`, `{"octocat/hello-world"}`, `{"*"}`, 1, testutils.AnyArgument{}, "octocat", 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + _, err := _sqlite.CreateSettings(context.TODO(), _settings) + if err != nil { + t.Errorf("unable to create test settings for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.UpdateSettings(context.TODO(), _settings) + got.SetUpdatedAt(_settings.GetUpdatedAt()) + + if test.failure { + if err == nil { + t.Errorf("UpdateSettings for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateSettings for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, _settings) { + t.Errorf("UpdateSettings for %s returned %s, want %s", test.name, got, _settings) + } + }) + } +} diff --git a/database/testutils/mock_args.go b/database/testutils/mock_args.go new file mode 100644 index 000000000..c2d8c562f --- /dev/null +++ b/database/testutils/mock_args.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package testutils + +import ( + "database/sql/driver" + "time" +) + +// This will be used with the github.com/DATA-DOG/go-sqlmock library to compare values +// that are otherwise not easily compared. These typically would be values generated +// before adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type AnyArgument struct{} + +// Match satisfies sqlmock.Argument interface. +func (a AnyArgument) Match(_ driver.Value) bool { + return true +} + +// NowTimestamp is used to test whether timestamps get updated correctly to the current time with lenience. +type NowTimestamp struct{} + +// Match satisfies sqlmock.Argument interface. +func (t NowTimestamp) Match(v driver.Value) bool { + ts, ok := v.(int64) + if !ok { + return false + } + + now := time.Now().Unix() + + return now-ts < 10 +} diff --git a/database/types/settings.go b/database/types/settings.go new file mode 100644 index 000000000..f17bac54b --- /dev/null +++ b/database/types/settings.go @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + + "github.com/lib/pq" + + "github.com/go-vela/server/api/types/settings" + "github.com/go-vela/server/util" +) + +var ( + // ErrEmptyCloneImage defines the error type when a + // Settings type has an empty CloneImage field provided. + ErrEmptyCloneImage = errors.New("empty settings clone image provided") +) + +type ( + // Platform is the database representation of platform settings. + Platform struct { + ID sql.NullInt64 `sql:"id"` + Compiler + Queue + + RepoAllowlist pq.StringArray `json:"repo_allowlist" sql:"repo_allowlist" gorm:"type:varchar(1000)"` + ScheduleAllowlist pq.StringArray `json:"schedule_allowlist" sql:"schedule_allowlist" gorm:"type:varchar(1000)"` + + CreatedAt sql.NullInt64 `sql:"created_at"` + UpdatedAt sql.NullInt64 `sql:"updated_at"` + UpdatedBy sql.NullString `sql:"updated_by"` + } + + // Compiler is the database representation of compiler settings. + Compiler struct { + CloneImage sql.NullString `json:"clone_image" sql:"clone_image"` + TemplateDepth sql.NullInt64 `json:"template_depth" sql:"template_depth"` + StarlarkExecLimit sql.NullInt64 `json:"starlark_exec_limit" sql:"starlark_exec_limit"` + } + + // Queue is the database representation of queue settings. + Queue struct { + Routes pq.StringArray `json:"routes" sql:"routes" gorm:"type:varchar(1000)"` + } +) + +// Value - Implementation of valuer for database/sql for Compiler. +func (r Compiler) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Compiler. +func (r *Compiler) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for compiler: %T", v) + } +} + +// Value - Implementation of valuer for database/sql for Queue. +func (r Queue) Value() (driver.Value, error) { + valueString, err := json.Marshal(r) + return string(valueString), err +} + +// Scan - Implement the database/sql scanner interface for Queue. +func (r *Queue) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, &r) + case string: + return json.Unmarshal([]byte(v), &r) + default: + return fmt.Errorf("wrong type for queue: %T", v) + } +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Settings type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (ps *Platform) Nullify() *Platform { + if ps == nil { + return nil + } + + // check if the ID field should be false + if ps.ID.Int64 == 0 { + ps.ID.Valid = false + } + + // check if the CloneImage field should be false + if len(ps.CloneImage.String) == 0 { + ps.CloneImage.Valid = false + } + + // check if the CreatedAt field should be false + if ps.CreatedAt.Int64 < 0 { + ps.CreatedAt.Valid = false + } + + // check if the UpdatedAt field should be false + if ps.UpdatedAt.Int64 < 0 { + ps.UpdatedAt.Valid = false + } + + return ps +} + +// ToAPI converts the Settings type +// to an API Settings type. +func (ps *Platform) ToAPI() *settings.Platform { + psAPI := new(settings.Platform) + psAPI.SetID(ps.ID.Int64) + + psAPI.SetRepoAllowlist(ps.RepoAllowlist) + psAPI.SetScheduleAllowlist(ps.ScheduleAllowlist) + + psAPI.Compiler = &settings.Compiler{} + psAPI.SetCloneImage(ps.CloneImage.String) + psAPI.SetTemplateDepth(int(ps.TemplateDepth.Int64)) + psAPI.SetStarlarkExecLimit(uint64(ps.StarlarkExecLimit.Int64)) + + psAPI.Queue = &settings.Queue{} + psAPI.SetRoutes(ps.Routes) + + psAPI.SetCreatedAt(ps.CreatedAt.Int64) + psAPI.SetUpdatedAt(ps.UpdatedAt.Int64) + psAPI.SetUpdatedBy(ps.UpdatedBy.String) + + return psAPI +} + +// Validate verifies the necessary fields for +// the Settings type are populated correctly. +func (ps *Platform) Validate() error { + // verify the CloneImage field is populated + if len(ps.CloneImage.String) == 0 { + return ErrEmptyCloneImage + } + + // verify compiler settings are within limits + if ps.TemplateDepth.Int64 <= 0 { + return fmt.Errorf("template depth must be greater than zero, got: %d", ps.TemplateDepth.Int64) + } + + if ps.StarlarkExecLimit.Int64 <= 0 { + return fmt.Errorf("starlark exec limit must be greater than zero, got: %d", ps.StarlarkExecLimit.Int64) + } + + // ensure that all Settings string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + ps.CloneImage = sql.NullString{String: util.Sanitize(ps.CloneImage.String), Valid: ps.CloneImage.Valid} + + // ensure that all Queue.Routes are sanitized + // to avoid unsafe HTML content + for i, v := range ps.Routes { + ps.Routes[i] = util.Sanitize(v) + } + + // ensure that all RepoAllowlist are sanitized + // to avoid unsafe HTML content + for i, v := range ps.RepoAllowlist { + ps.RepoAllowlist[i] = util.Sanitize(v) + } + + // ensure that all ScheduleAllowlist are sanitized + // to avoid unsafe HTML content + for i, v := range ps.ScheduleAllowlist { + ps.ScheduleAllowlist[i] = util.Sanitize(v) + } + + if ps.CreatedAt.Int64 < 0 { + return fmt.Errorf("created_at must be greater than zero, got: %d", ps.CreatedAt.Int64) + } + + if ps.UpdatedAt.Int64 < 0 { + return fmt.Errorf("updated_at must be greater than zero, got: %d", ps.UpdatedAt.Int64) + } + + return nil +} + +// FromAPI converts the API Settings type +// to a database Settings type. +func FromAPI(s *settings.Platform) *Platform { + settings := &Platform{ + ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: s.GetCloneImage(), Valid: true}, + TemplateDepth: sql.NullInt64{Int64: int64(s.GetTemplateDepth()), Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: int64(s.GetStarlarkExecLimit()), Valid: true}, + }, + Queue: Queue{ + Routes: pq.StringArray(s.GetRoutes()), + }, + RepoAllowlist: pq.StringArray(s.GetRepoAllowlist()), + ScheduleAllowlist: pq.StringArray(s.GetScheduleAllowlist()), + CreatedAt: sql.NullInt64{Int64: s.GetCreatedAt(), Valid: true}, + UpdatedAt: sql.NullInt64{Int64: s.GetUpdatedAt(), Valid: true}, + UpdatedBy: sql.NullString{String: s.GetUpdatedBy(), Valid: true}, + } + + return settings.Nullify() +} diff --git a/database/types/settings_test.go b/database/types/settings_test.go new file mode 100644 index 000000000..973e5d88d --- /dev/null +++ b/database/types/settings_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "reflect" + "testing" + + api "github.com/go-vela/server/api/types/settings" +) + +func TestTypes_Platform_Nullify(t *testing.T) { + // setup types + var ps *Platform + + want := &Platform{ + ID: sql.NullInt64{Int64: 0, Valid: false}, + } + + // setup tests + tests := []struct { + repo *Platform + want *Platform + }{ + { + repo: testPlatform(), + want: testPlatform(), + }, + { + repo: ps, + want: nil, + }, + { + repo: new(Platform), + want: want, + }, + } + + // run tests + for _, test := range tests { + got := test.repo.Nullify() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("Nullify is %v, want %v", got, test.want) + } + } +} + +func TestTypes_Platform_ToAPI(t *testing.T) { + // setup types + want := new(api.Platform) + want.SetID(1) + want.SetRepoAllowlist([]string{"github/octocat"}) + want.SetScheduleAllowlist([]string{"*"}) + want.SetCreatedAt(0) + want.SetUpdatedAt(0) + want.SetUpdatedBy("") + + want.Compiler = new(api.Compiler) + want.SetCloneImage("target/vela-git:latest") + want.SetTemplateDepth(10) + want.SetStarlarkExecLimit(100) + + want.Queue = new(api.Queue) + want.SetRoutes([]string{"vela"}) + + // run test + got := testPlatform().ToAPI() + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestTypes_Platform_Validate(t *testing.T) { + // setup tests + tests := []struct { + failure bool + settings *Platform + }{ + { + failure: false, + settings: testPlatform(), + }, + { // no CloneImage set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + }, + }, + { // no TemplateDepth set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + }, + }, + { // no StarlarkExecLimit set for settings + failure: true, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + }, + }, + }, + { // no queue fields set for settings + failure: false, + settings: &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + Queue: Queue{}, + }, + }, + } + + // run tests + for _, test := range tests { + err := test.settings.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} + +func TestTypes_Platform_PlatformFromAPI(t *testing.T) { + // setup types + s := new(api.Platform) + s.SetID(1) + s.SetRepoAllowlist([]string{"github/octocat"}) + s.SetScheduleAllowlist([]string{"*"}) + s.SetCreatedAt(0) + s.SetUpdatedAt(0) + s.SetUpdatedBy("") + + s.Compiler = new(api.Compiler) + s.SetCloneImage("target/vela-git:latest") + s.SetTemplateDepth(10) + s.SetStarlarkExecLimit(100) + + s.Queue = new(api.Queue) + s.SetRoutes([]string{"vela"}) + + want := testPlatform() + + // run test + got := FromAPI(s) + + if !reflect.DeepEqual(got, want) { + t.Errorf("PlatformFromAPI is %v, want %v", got, want) + } +} + +// testPlatform is a test helper function to create a Platform +// type with all fields set to a fake value. +func testPlatform() *Platform { + return &Platform{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Compiler: Compiler{ + CloneImage: sql.NullString{String: "target/vela-git:latest", Valid: true}, + TemplateDepth: sql.NullInt64{Int64: 10, Valid: true}, + StarlarkExecLimit: sql.NullInt64{Int64: 100, Valid: true}, + }, + Queue: Queue{ + Routes: []string{"vela"}, + }, + RepoAllowlist: []string{"github/octocat"}, + ScheduleAllowlist: []string{"*"}, + CreatedAt: sql.NullInt64{Int64: 0, Valid: true}, + UpdatedAt: sql.NullInt64{Int64: 0, Valid: true}, + UpdatedBy: sql.NullString{String: "", Valid: true}, + } +} diff --git a/go.mod b/go.mod index aefa64a29..4993597bb 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/alicebob/miniredis/v2 v2.32.1 github.com/aws/aws-sdk-go v1.51.0 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 + github.com/distribution/reference v0.6.0 github.com/drone/envsubst v1.0.3 github.com/ghodss/yaml v1.0.0 github.com/gin-gonic/gin v1.9.1 @@ -99,6 +100,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect diff --git a/go.sum b/go.sum index 3b30dd8ed..75bac3763 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -213,6 +215,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/image/doc.go b/internal/image/doc.go new file mode 100644 index 000000000..66d473d15 --- /dev/null +++ b/internal/image/doc.go @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package image provides the ability for Vela to manage +// and manipulate images. +// +// Usage: +// +// import "github.com/go-vela/server/internal/image" +package image diff --git a/internal/image/image.go b/internal/image/image.go new file mode 100644 index 000000000..daa5b37bc --- /dev/null +++ b/internal/image/image.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "github.com/distribution/reference" +) + +// ParseWithError digests the provided image into a +// fully qualified canonical reference. If an error +// occurs, it will return the last digested form of +// the image. +func ParseWithError(_image string) (string, error) { + // parse the image provided into a + // named, fully qualified reference + // + // https://pkg.go.dev/github.com/distribution/reference#ParseAnyReference + _reference, err := reference.ParseAnyReference(_image) + if err != nil { + return _image, err + } + + // ensure we have the canonical form of the named reference + // + // https://pkg.go.dev/github.com/distribution/reference#ParseNamed + _canonical, err := reference.ParseNamed(_reference.String()) + if err != nil { + return _reference.String(), err + } + + // ensure the canonical reference has a tag + // + // https://pkg.go.dev/github.com/distribution/reference#TagNameOnly + return reference.TagNameOnly(_canonical).String(), nil +} diff --git a/internal/image/image_test.go b/internal/image/image_test.go new file mode 100644 index 000000000..365fad2ff --- /dev/null +++ b/internal/image/image_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 + +package image + +import ( + "strings" + "testing" +) + +func TestImage_ParseWithError(t *testing.T) { + // setup tests + tests := []struct { + name string + failure bool + image string + want string + }{ + { + name: "image only", + failure: false, + image: "golang", + want: "docker.io/library/golang:latest", + }, + { + name: "image and tag", + failure: false, + image: "golang:latest", + want: "docker.io/library/golang:latest", + }, + { + name: "image and tag", + failure: false, + image: "golang:1.14", + want: "docker.io/library/golang:1.14", + }, + { + name: "fails with bad image", + failure: true, + image: "!@#$%^&*()", + want: "!@#$%^&*()", + }, + { + name: "fails with image sha", + failure: true, + image: "1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + want: "sha256:1a3f5e7d9c1b3a5f7e9d1c3b5a7f9e1d3c5b7a9f1e3d5d7c9b1a3f5e7d9c1b3a", + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := ParseWithError(test.image) + + if test.failure { + if err == nil { + t.Errorf("ParseWithError should have returned err") + } + + if !strings.EqualFold(got, test.want) { + t.Errorf("ParseWithError is %s want %s", got, test.want) + } + + return // continue to next test + } + + if err != nil { + t.Errorf("ParseWithError returned err: %v", err) + } + + if !strings.EqualFold(got, test.want) { + t.Errorf("ParseWithError is %s want %s", got, test.want) + } + }) + } +} diff --git a/mock/server/server.go b/mock/server/server.go index fb6161ccd..9527cbc75 100644 --- a/mock/server/server.go +++ b/mock/server/server.go @@ -29,6 +29,9 @@ func FakeHandler() http.Handler { e.PUT("/api/v1/admin/user", updateUser) e.POST("/api/v1/admin/workers/:worker/register", registerToken) e.PUT("api/v1/admin/clean", cleanResoures) + e.GET("/api/v1/admin/settings", getSettings) + e.PUT("/api/v1/admin/settings", updateSettings) + e.DELETE("/api/v1/admin/settings", restoreSettings) // mock endpoints for build calls e.GET("/api/v1/repos/:org/:repo/builds/:build", getBuild) diff --git a/mock/server/settings.go b/mock/server/settings.go new file mode 100644 index 000000000..07e5c9419 --- /dev/null +++ b/mock/server/settings.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/types/settings" +) + +const ( + // SettingsResp represents a JSON return for a single settings. + SettingsResp = ` + { + "id": 1, + "compiler": { + "clone_image": "target/vela-git", + "template_depth": 3, + "starlark_exec_limit": 100 + }, + "queue": { + "routes": [ + "vela" + ] + }, + "repo_allowlist": [ + "*" + ], + "schedule_allowlist": [ + "octocat/hello-world" + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" + }` + + // UpdateSettingsResp represents a JSON return for modifying a settings field. + UpdateSettingsResp = ` + { + "id": 1, + "compiler": { + "clone_image": "target/vela-git:latest", + "template_depth": 5, + "starlark_exec_limit": 123 + }, + "queue": { + "routes": [ + "vela", + "large" + ] + }, + "repo_allowlist": [], + "schedule_allowlist": [ + "octocat/hello-world", + "octocat/*" + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" + }` + + // RestoreSettingsResp represents a JSON return for restoring the settings record to the defaults. + RestoreSettingsResp = ` + { + "id": 1, + "compiler": { + "clone_image": "target/vela-git:latest", + "template_depth": 5, + "starlark_exec_limit": 123 + }, + "queue": { + "routes": [ + "vela", + "large" + ] + }, + "repo_allowlist": [], + "schedule_allowlist": [ + "octocat/hello-world", + "octocat/*" + ], + "created_at": 1, + "updated_at": 1, + "updated_by": "octocat" + }` +) + +// getSettings has a param :settings returns mock JSON for a http GET. +func getSettings(c *gin.Context) { + data := []byte(SettingsResp) + + var body settings.Platform + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// updateSettings returns mock JSON for a http PUT. +func updateSettings(c *gin.Context) { + data := []byte(UpdateSettingsResp) + + var body settings.Platform + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} + +// restoreSettings returns mock JSON for a http DELETE. +func restoreSettings(c *gin.Context) { + data := []byte(RestoreSettingsResp) + + var body settings.Platform + _ = json.Unmarshal(data, &body) + + c.JSON(http.StatusOK, body) +} diff --git a/mock/server/settings_test.go b/mock/server/settings_test.go new file mode 100644 index 000000000..d827cc756 --- /dev/null +++ b/mock/server/settings_test.go @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: Apache-2.0 + +package server + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_GetResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(SettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("SettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} + +func TestSettings_UpdateResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(UpdateSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("UpdateSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} + +func TestSettings_RestoreResp(t *testing.T) { + testSettings := settings.Platform{} + + err := json.Unmarshal([]byte(RestoreSettingsResp), &testSettings) + if err != nil { + t.Errorf("error unmarshaling settings: %v", err) + } + + tSettings := reflect.TypeOf(testSettings) + + for i := 0; i < tSettings.NumField(); i++ { + f := reflect.ValueOf(testSettings).Field(i) + if f.IsNil() { + t.Errorf("RestoreSettingsResp missing field %s", tSettings.Field(i).Name) + } + } +} diff --git a/queue/queue.go b/queue/queue.go index fe2946f1b..181ed5899 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -6,10 +6,32 @@ import ( "fmt" "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" "github.com/go-vela/types/constants" ) +// FromCLIContext helper function to setup the queue from the CLI arguments. +func FromCLIContext(c *cli.Context) (Service, error) { + logrus.Debug("Creating queue client from CLI configuration") + + // queue configuration + _setup := &Setup{ + Driver: c.String("queue.driver"), + Address: c.String("queue.addr"), + Cluster: c.Bool("queue.cluster"), + Routes: c.StringSlice("queue.routes"), + Timeout: c.Duration("queue.pop.timeout"), + PrivateKey: c.String("queue.private-key"), + PublicKey: c.String("queue.public-key"), + } + + // setup the queue + // + // https://pkg.go.dev/github.com/go-vela/server/queue?tab=doc#New + return New(_setup) +} + // New creates and returns a Vela service capable of // integrating with the configured queue environment. // Currently, the following queues are supported: diff --git a/queue/redis/driver_test.go b/queue/redis/driver_test.go index 8ed67a3a5..52894f1ba 100644 --- a/queue/redis/driver_test.go +++ b/queue/redis/driver_test.go @@ -28,7 +28,7 @@ func TestRedis_Driver(t *testing.T) { _service, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/length.go b/queue/redis/length.go index c4d7bb7c3..7f5ed9b0b 100644 --- a/queue/redis/length.go +++ b/queue/redis/length.go @@ -6,13 +6,13 @@ import ( "context" ) -// Length tallies all items present in the configured channels in the queue. +// Length tallies all items present in the configured routes in the queue. func (c *client) Length(ctx context.Context) (int64, error) { - c.Logger.Tracef("reading length of all configured channels in queue") + c.Logger.Tracef("reading length of all configured routes in queue") total := int64(0) - for _, channel := range c.config.Channels { + for _, channel := range c.GetRoutes() { items, err := c.Redis.LLen(ctx, channel).Result() if err != nil { return 0, err diff --git a/queue/redis/length_test.go b/queue/redis/length_test.go index 545f9f54a..9656a8c20 100644 --- a/queue/redis/length_test.go +++ b/queue/redis/length_test.go @@ -33,26 +33,26 @@ func TestRedis_Length(t *testing.T) { // setup tests tests := []struct { - channels []string - want int64 + routes []string + want int64 }{ { - channels: []string{"vela"}, - want: 1, + routes: []string{"vela"}, + want: 1, }, { - channels: []string{"vela", "vela:second", "vela:third"}, - want: 4, + routes: []string{"vela", "vela:second", "vela:third"}, + want: 4, }, { - channels: []string{"vela", "vela:second", "phony"}, - want: 6, + routes: []string{"vela", "vela:second", "phony"}, + want: 6, }, } // run tests for _, test := range tests { - for _, channel := range test.channels { + for _, channel := range test.routes { err := _redis.Push(context.Background(), channel, bytes) if err != nil { t.Errorf("unable to push item to queue: %v", err) diff --git a/queue/redis/opts.go b/queue/redis/opts.go index 582994fea..f58dab06a 100644 --- a/queue/redis/opts.go +++ b/queue/redis/opts.go @@ -29,18 +29,18 @@ func WithAddress(address string) ClientOpt { } } -// WithChannels sets the channels in the queue client for Redis. -func WithChannels(channels ...string) ClientOpt { +// WithRoutes sets the routes in the queue client for Redis. +func WithRoutes(routes ...string) ClientOpt { return func(c *client) error { - c.Logger.Trace("configuring channels in redis queue client") + c.Logger.Trace("configuring routes in redis queue client") - // check if the channels provided are empty - if len(channels) == 0 { - return fmt.Errorf("no Redis queue channels provided") + // check if the routes provided are empty + if len(routes) == 0 { + return fmt.Errorf("no Redis queue routes provided") } - // set the queue channels in the redis client - c.config.Channels = channels + // set the queue routes in the redis client + c.SetRoutes(routes) return nil } diff --git a/queue/redis/opts_test.go b/queue/redis/opts_test.go index aa775c121..6e729ce52 100644 --- a/queue/redis/opts_test.go +++ b/queue/redis/opts_test.go @@ -64,7 +64,7 @@ func TestRedis_ClientOpt_WithAddress(t *testing.T) { } } -func TestRedis_ClientOpt_WithChannels(t *testing.T) { +func TestRedis_ClientOpt_WithRoutes(t *testing.T) { // setup tests // create a local fake redis instance // @@ -76,19 +76,19 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { defer _redis.Close() tests := []struct { - failure bool - channels []string - want []string + failure bool + routes []string + want []string }{ { - failure: false, - channels: []string{"foo", "bar"}, - want: []string{"foo", "bar"}, + failure: false, + routes: []string{"foo", "bar"}, + want: []string{"foo", "bar"}, }, { - failure: true, - channels: []string{}, - want: []string{}, + failure: true, + routes: []string{}, + want: []string{}, }, } @@ -96,23 +96,23 @@ func TestRedis_ClientOpt_WithChannels(t *testing.T) { for _, test := range tests { _service, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels(test.channels...), + WithRoutes(test.routes...), ) if test.failure { if err == nil { - t.Errorf("WithChannels should have returned err") + t.Errorf("WithRoutes should have returned err") } continue } if err != nil { - t.Errorf("WithChannels returned err: %v", err) + t.Errorf("WithRoutes returned err: %v", err) } - if !reflect.DeepEqual(_service.config.Channels, test.want) { - t.Errorf("WithChannels is %v, want %v", _service.config.Channels, test.want) + if !reflect.DeepEqual(_service.GetRoutes(), test.want) { + t.Errorf("WithRoutes is %v, want %v", _service.GetRoutes(), test.want) } } } diff --git a/queue/redis/ping_test.go b/queue/redis/ping_test.go index d069370f2..39fde9272 100644 --- a/queue/redis/ping_test.go +++ b/queue/redis/ping_test.go @@ -22,7 +22,7 @@ func TestRedis_Ping_Good(t *testing.T) { // setup redis mock goodRedis, err := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) @@ -49,7 +49,7 @@ func TestRedis_Ping_Bad(t *testing.T) { // setup redis mock badRedis, _ := New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/pop.go b/queue/redis/pop.go index 4d1af3921..7c5658029 100644 --- a/queue/redis/pop.go +++ b/queue/redis/pop.go @@ -14,23 +14,23 @@ import ( ) // Pop grabs an item from the specified channel off the queue. -func (c *client) Pop(ctx context.Context, routes []string) (*models.Item, error) { - c.Logger.Tracef("popping item from queue %s", c.config.Channels) +func (c *client) Pop(ctx context.Context, inRoutes []string) (*models.Item, error) { + c.Logger.Tracef("popping item from queue %s", c.GetRoutes()) - // define channels to pop from - var channels []string + // define routes to pop from + var routes []string // if routes were supplied, use those if len(routes) > 0 { - channels = routes + routes = inRoutes } else { - channels = c.config.Channels + routes = c.GetRoutes() } // build a redis queue command to pop an item from queue // // https://pkg.go.dev/github.com/go-redis/redis?tab=doc#Client.BLPop - popCmd := c.Redis.BLPop(ctx, c.config.Timeout, channels...) + popCmd := c.Redis.BLPop(ctx, c.config.Timeout, routes...) // blocking call to pop item from queue // diff --git a/queue/redis/pop_test.go b/queue/redis/pop_test.go index b2ade0b40..190839f80 100644 --- a/queue/redis/pop_test.go +++ b/queue/redis/pop_test.go @@ -64,7 +64,7 @@ func TestRedis_Pop(t *testing.T) { t.Errorf("unable to create queue service: %v", err) } // overwrite channel to be invalid - badChannel.config.Channels = nil + badChannel.SetRoutes(nil) signed = sign.Sign(out, bytes, badChannel.config.PrivateKey) @@ -76,10 +76,10 @@ func TestRedis_Pop(t *testing.T) { // setup tests tests := []struct { - failure bool - redis *client - want *models.Item - channels []string + failure bool + redis *client + want *models.Item + routes []string }{ { failure: false, @@ -87,10 +87,10 @@ func TestRedis_Pop(t *testing.T) { want: _item, }, { - failure: false, - redis: _redis, - want: _item, - channels: []string{"custom"}, + failure: false, + redis: _redis, + want: _item, + routes: []string{"custom"}, }, { failure: false, @@ -106,7 +106,7 @@ func TestRedis_Pop(t *testing.T) { // run tests for _, test := range tests { - got, err := test.redis.Pop(context.Background(), test.channels) + got, err := test.redis.Pop(context.Background(), test.routes) if test.failure { if err == nil { diff --git a/queue/redis/redis.go b/queue/redis/redis.go index 5a789ed58..88b962c00 100644 --- a/queue/redis/redis.go +++ b/queue/redis/redis.go @@ -11,13 +11,13 @@ import ( "github.com/alicebob/miniredis/v2" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" + + "github.com/go-vela/server/api/types/settings" ) type config struct { // specifies the address to use for the Redis client Address string - // specifies a list of channels for managing builds for the Redis client - Channels []string // enables the Redis client to integrate with a Redis cluster Cluster bool // specifies the timeout to use for the Redis client @@ -32,6 +32,9 @@ type client struct { config *config Redis *redis.Client Options *redis.Options + + settings.Queue + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -174,7 +177,7 @@ func pingQueue(c *client) error { // This function is intended for running tests only. // //nolint:revive // ignore returning unexported client -func NewTest(signingPrivateKey, signingPublicKey string, channels ...string) (*client, error) { +func NewTest(signingPrivateKey, signingPublicKey string, routes ...string) (*client, error) { // create a local fake redis instance // // https://pkg.go.dev/github.com/alicebob/miniredis/v2#Run @@ -185,7 +188,7 @@ func NewTest(signingPrivateKey, signingPublicKey string, channels ...string) (*c return New( WithAddress(fmt.Sprintf("redis://%s", _redis.Addr())), - WithChannels(channels...), + WithRoutes(routes...), WithCluster(false), WithPrivateKey(signingPrivateKey), WithPublicKey(signingPublicKey), diff --git a/queue/redis/redis_test.go b/queue/redis/redis_test.go index 474d57de9..b08fda760 100644 --- a/queue/redis/redis_test.go +++ b/queue/redis/redis_test.go @@ -125,7 +125,7 @@ func TestRedis_New(t *testing.T) { for _, test := range tests { _, err := New( WithAddress(test.address), - WithChannels("foo"), + WithRoutes("foo"), WithCluster(false), WithTimeout(5*time.Second), ) diff --git a/queue/redis/route.go b/queue/redis/route.go index a25a537c6..e4fc0ae05 100644 --- a/queue/redis/route.go +++ b/queue/redis/route.go @@ -13,7 +13,7 @@ import ( // Route decides which route a build gets placed within the queue. func (c *client) Route(w *pipeline.Worker) (string, error) { - c.Logger.Tracef("deciding route from queue channels %s", c.config.Channels) + c.Logger.Tracef("deciding route from queue routes %s", c.GetRoutes()) // create buffer to store route buf := bytes.Buffer{} @@ -37,7 +37,7 @@ func (c *client) Route(w *pipeline.Worker) (string, error) { route := strings.TrimLeft(buf.String(), ":") - for _, r := range c.config.Channels { + for _, r := range c.GetRoutes() { if strings.EqualFold(route, r) { return route, nil } diff --git a/queue/redis/settings.go b/queue/redis/settings.go new file mode 100644 index 000000000..70747b6c3 --- /dev/null +++ b/queue/redis/settings.go @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: Apache-2.0 + +package redis + +import ( + "github.com/go-vela/server/api/types/settings" +) + +// GetSettings retrieves the api settings type in the Engine. +func (c *client) GetSettings() settings.Queue { + return c.Queue +} + +// SetSettings sets the api settings type in the Engine. +func (c *client) SetSettings(s *settings.Platform) { + if s != nil { + c.SetRoutes(s.GetRoutes()) + } +} diff --git a/queue/service.go b/queue/service.go index 3fdf27515..a976e7045 100644 --- a/queue/service.go +++ b/queue/service.go @@ -5,6 +5,7 @@ package queue import ( "context" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/queue/models" "github.com/go-vela/types/pipeline" ) @@ -37,4 +38,12 @@ type Service interface { // Route defines a function that decides which // channel a build gets placed within the queue. Route(*pipeline.Worker) (string, error) + + // GetSettings defines a function that returns + // queue settings. + GetSettings() settings.Queue + + // SetSettings defines a function that takes api settings + // and updates the compiler Engine. + SetSettings(*settings.Platform) } diff --git a/queue/setup.go b/queue/setup.go index cdac822ea..c554c9238 100644 --- a/queue/setup.go +++ b/queue/setup.go @@ -45,7 +45,7 @@ func (s *Setup) Redis() (Service, error) { // https://pkg.go.dev/github.com/go-vela/server/queue/redis?tab=doc#New return redis.New( redis.WithAddress(s.Address), - redis.WithChannels(s.Routes...), + redis.WithRoutes(s.Routes...), redis.WithCluster(s.Cluster), redis.WithTimeout(s.Timeout), redis.WithPrivateKey(s.PrivateKey), diff --git a/router/admin.go b/router/admin.go index c279445eb..22f83973e 100644 --- a/router/admin.go +++ b/router/admin.go @@ -12,18 +12,21 @@ import ( // AdminHandlers is a function that extends the provided base router group // with the API handlers for admin functionality. // -// GET /api/v1/admin/builds/queue -// GET /api/v1/admin/build/:id -// PUT /api/v1/admin/build -// PUT /api/v1/admin/clean -// PUT /api/v1/admin/deployment -// PUT /api/v1/admin/hook -// PUT /api/v1/admin/repo -// PUT /api/v1/admin/secret -// PUT /api/v1/admin/service -// PUT /api/v1/admin/step -// PUT /api/v1/admin/user -// POST /api/v1/admin/workers/:worker/register. +// GET /api/v1/admin/builds/queue +// GET /api/v1/admin/build/:id +// PUT /api/v1/admin/build +// PUT /api/v1/admin/clean +// PUT /api/v1/admin/deployment +// PUT /api/v1/admin/hook +// PUT /api/v1/admin/repo +// PUT /api/v1/admin/secret +// PUT /api/v1/admin/service +// PUT /api/v1/admin/step +// PUT /api/v1/admin/user +// POST /api/v1/admin/workers/:worker/register +// GET /api/v1/admin/settings +// PUT /api/v1/admin/settings +// DELETE /api/v1/admin/settings. func AdminHandlers(base *gin.RouterGroup) { // Admin endpoints _admin := base.Group("/admin", perm.MustPlatformAdmin()) @@ -60,5 +63,10 @@ func AdminHandlers(base *gin.RouterGroup) { // Admin worker endpoint _admin.POST("/workers/:worker/register", admin.RegisterToken) + + // Admin settings endpoints + _admin.GET("/settings", admin.GetSettings) + _admin.PUT("/settings", admin.UpdateSettings) + _admin.DELETE("/settings", admin.RestoreSettings) } // end of admin endpoints } diff --git a/router/middleware/allowlist.go b/router/middleware/allowlist.go deleted file mode 100644 index edfe5feb2..000000000 --- a/router/middleware/allowlist.go +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "github.com/gin-gonic/gin" -) - -// Allowlist is a middleware function that attaches the allowlist used -// to limit which repos can be activated within the system. -func Allowlist(allowlist []string) gin.HandlerFunc { - return func(c *gin.Context) { - c.Set("allowlist", allowlist) - c.Next() - } -} diff --git a/router/middleware/allowlist_schedule.go b/router/middleware/allowlist_schedule.go deleted file mode 100644 index d22a7aa20..000000000 --- a/router/middleware/allowlist_schedule.go +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -package middleware - -import ( - "github.com/gin-gonic/gin" -) - -// AllowlistSchedule is a middleware function that attaches the allowlistschedule used -// to limit which repos can utilize the schedule feature within the system. -func AllowlistSchedule(allowlistschedule []string) gin.HandlerFunc { - return func(c *gin.Context) { - c.Set("allowlistschedule", allowlistschedule) - c.Next() - } -} diff --git a/router/middleware/cli.go b/router/middleware/cli.go new file mode 100644 index 000000000..acc1972eb --- /dev/null +++ b/router/middleware/cli.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" + + cliMiddleware "github.com/go-vela/server/router/middleware/cli" +) + +// CLI is a middleware function that attaches the cli client +// to the context of every http.Request. +func CLI(cliCtx *cli.Context) gin.HandlerFunc { + return func(c *gin.Context) { + cliMiddleware.ToContext(c, cliCtx) + + c.Next() + } +} diff --git a/router/middleware/cli/context.go b/router/middleware/cli/context.go new file mode 100644 index 000000000..13952ddc2 --- /dev/null +++ b/router/middleware/cli/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "context" + + "github.com/urfave/cli/v2" +) + +const key = "cli" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the cli context associated with this context. +func FromContext(c context.Context) *cli.Context { + value := c.Value(key) + if value == nil { + return nil + } + + s, ok := value.(*cli.Context) + if !ok { + return nil + } + + return s +} + +// ToContext adds the cli context to this context if it supports +// the Setter interface. +func ToContext(c Setter, s *cli.Context) { + c.Set(key, s) +} diff --git a/router/middleware/cli/context_test.go b/router/middleware/cli/context_test.go new file mode 100644 index 000000000..4269f3c06 --- /dev/null +++ b/router/middleware/cli/context_test.go @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" +) + +func TestSettings_FromContext(t *testing.T) { + // setup types + want := &cli.Context{} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestSettings_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_ToContext(t *testing.T) { + // setup types + want := &cli.Context{} + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/cli/doc.go b/router/middleware/cli/doc.go new file mode 100644 index 000000000..3bdc404c6 --- /dev/null +++ b/router/middleware/cli/doc.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package cli provides the ability for inserting +// Vela cli resources into or extracting Vela cli +// resources from the middleware chain for the API. +// +// Usage: +// +// import "github.com/go-vela/server/router/middleware/cli" +package cli diff --git a/router/middleware/allowlist_schedule_test.go b/router/middleware/cli_test.go similarity index 65% rename from router/middleware/allowlist_schedule_test.go rename to router/middleware/cli_test.go index 1461b8719..d94e68c75 100644 --- a/router/middleware/allowlist_schedule_test.go +++ b/router/middleware/cli_test.go @@ -9,12 +9,18 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/urfave/cli/v2" ) -func TestMiddleware_AllowlistSchedule(t *testing.T) { +func TestMiddleware_CLI(t *testing.T) { // setup types - got := []string{""} - want := []string{"foobar"} + want := &cli.Context{ + App: &cli.App{ + Name: "foo", + }, + } + + got := &cli.Context{} // setup context gin.SetMode(gin.TestMode) @@ -24,9 +30,9 @@ func TestMiddleware_AllowlistSchedule(t *testing.T) { context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) // setup mock server - engine.Use(AllowlistSchedule(want)) + engine.Use(CLI(want)) engine.GET("/health", func(c *gin.Context) { - got = c.Value("allowlistschedule").([]string) + got = c.Value("cli").(*cli.Context) c.Status(http.StatusOK) }) @@ -35,10 +41,10 @@ func TestMiddleware_AllowlistSchedule(t *testing.T) { engine.ServeHTTP(context.Writer, context.Request) if resp.Code != http.StatusOK { - t.Errorf("AllowlistSchedule returned %v, want %v", resp.Code, http.StatusOK) + t.Errorf("CLI returned %v, want %v", resp.Code, http.StatusOK) } if !reflect.DeepEqual(got, want) { - t.Errorf("AllowlistSchedule is %v, want %v", got, want) + t.Errorf("CLI is %v, want %v", got, want) } } diff --git a/router/middleware/compiler.go b/router/middleware/compiler.go index be25a9992..fbd382a4d 100644 --- a/router/middleware/compiler.go +++ b/router/middleware/compiler.go @@ -6,13 +6,18 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/compiler" + "github.com/go-vela/server/router/middleware/settings" ) // Compiler is a middleware function that initializes the compiler and // attaches to the context of every http.Request. -func Compiler(cli compiler.Engine) gin.HandlerFunc { +func Compiler(comp compiler.Engine) gin.HandlerFunc { return func(c *gin.Context) { - compiler.WithGinContext(c, cli) + s := settings.FromContext(c) + comp.SetSettings(s) + + compiler.WithGinContext(c, comp) + c.Next() } } diff --git a/router/middleware/compiler_test.go b/router/middleware/compiler_test.go index 517de8881..d018b06a9 100644 --- a/router/middleware/compiler_test.go +++ b/router/middleware/compiler_test.go @@ -12,25 +12,50 @@ import ( "github.com/gin-gonic/gin" "github.com/urfave/cli/v2" + "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/native" + sMiddleware "github.com/go-vela/server/router/middleware/settings" ) func TestMiddleware_CompilerNative(t *testing.T) { // setup types - var got compiler.Engine + defaultCloneImage := "target/vela-git" + wantCloneImage := "target/vela-git:latest" + + set := flag.NewFlagSet("", flag.ExitOnError) + set.String("clone-image", defaultCloneImage, "doc") - want, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want, _ := native.FromCLIContext(cli.NewContext(nil, set, nil)) + want.SetCloneImage(wantCloneImage) + + var got compiler.Engine + got, _ = native.FromCLIContext(cli.NewContext(nil, set, nil)) // setup context gin.SetMode(gin.TestMode) resp := httptest.NewRecorder() context, engine := gin.CreateTestContext(resp) + + engine.Use(func() gin.HandlerFunc { + return func(c *gin.Context) { + s := settings.Platform{ + Compiler: &settings.Compiler{}, + } + s.SetCloneImage(wantCloneImage) + + sMiddleware.ToContext(c, &s) + + c.Next() + } + }(), + ) + + engine.Use(Compiler(got)) + context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) - // setup mock server - engine.Use(Compiler(want)) engine.GET("/health", func(c *gin.Context) { got = compiler.FromContext(c) diff --git a/router/middleware/pipeline/pipeline_test.go b/router/middleware/pipeline/pipeline_test.go index 489025859..d7af22303 100644 --- a/router/middleware/pipeline/pipeline_test.go +++ b/router/middleware/pipeline/pipeline_test.go @@ -285,7 +285,7 @@ func TestPipeline_Establish_NoPipeline(t *testing.T) { set := flag.NewFlagSet("test", 0) set.String("clone-image", "target/vela-git:latest", "doc") - comp, err := native.New(cli.NewContext(nil, set, nil)) + comp, err := native.FromCLIContext(cli.NewContext(nil, set, nil)) if err != nil { t.Errorf("unable to create compiler: %v", err) } diff --git a/router/middleware/queue.go b/router/middleware/queue.go index 7f157ccb3..eff077aa0 100644 --- a/router/middleware/queue.go +++ b/router/middleware/queue.go @@ -6,13 +6,18 @@ import ( "github.com/gin-gonic/gin" "github.com/go-vela/server/queue" + "github.com/go-vela/server/router/middleware/settings" ) // Queue is a middleware function that initializes the queue and // attaches to the context of every http.Request. func Queue(q queue.Service) gin.HandlerFunc { return func(c *gin.Context) { + s := settings.FromContext(c) + q.SetSettings(s) + queue.WithGinContext(c, q) + c.Next() } } diff --git a/router/middleware/settings.go b/router/middleware/settings.go new file mode 100644 index 000000000..1e985bf73 --- /dev/null +++ b/router/middleware/settings.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +package middleware + +import ( + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/types/settings" + sMiddleware "github.com/go-vela/server/router/middleware/settings" +) + +// Settings is a middleware function that attaches settings +// to the context of every http.Request. +func Settings(s *settings.Platform) gin.HandlerFunc { + return func(c *gin.Context) { + sMiddleware.ToContext(c, s) + + c.Next() + } +} diff --git a/router/middleware/settings/context.go b/router/middleware/settings/context.go new file mode 100644 index 000000000..f5aa5fb62 --- /dev/null +++ b/router/middleware/settings/context.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "context" + + "github.com/go-vela/server/api/types/settings" +) + +const key = "settings" + +// Setter defines a context that enables setting values. +type Setter interface { + Set(string, interface{}) +} + +// FromContext returns the Settings associated with this context. +func FromContext(c context.Context) *settings.Platform { + value := c.Value(key) + if value == nil { + return nil + } + + s, ok := value.(*settings.Platform) + if !ok { + return nil + } + + return s +} + +// ToContext adds the Settings to this context if it supports +// the Setter interface. +func ToContext(c Setter, s *settings.Platform) { + c.Set(key, s) +} diff --git a/router/middleware/settings/context_test.go b/router/middleware/settings/context_test.go new file mode 100644 index 000000000..0c17b4297 --- /dev/null +++ b/router/middleware/settings/context_test.go @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: Apache-2.0 + +package settings + +import ( + "testing" + + "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/types/settings" +) + +func TestSettings_FromContext(t *testing.T) { + // setup types + num := int64(1) + cloneImage := "target/vela-git" + + cs := settings.Compiler{ + CloneImage: &cloneImage, + } + + want := &settings.Platform{ + ID: &num, + Compiler: &cs, + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, want) + + // run test + got := FromContext(context) + + if got != want { + t.Errorf("FromContext is %v, want %v", got, want) + } +} + +func TestSettings_FromContext_Bad(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_WrongType(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + context.Set(key, 1) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_FromContext_Empty(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + + // run test + got := FromContext(context) + + if got != nil { + t.Errorf("FromContext is %v, want nil", got) + } +} + +func TestSettings_ToContext(t *testing.T) { + // setup types + num := int64(1) + cloneImage := "target/vela-git" + + cs := settings.Compiler{ + CloneImage: &cloneImage, + } + + want := &settings.Platform{ + ID: &num, + Compiler: &cs, + } + + // setup context + gin.SetMode(gin.TestMode) + context, _ := gin.CreateTestContext(nil) + ToContext(context, want) + + // run test + got := context.Value(key) + + if got != want { + t.Errorf("ToContext is %v, want %v", got, want) + } +} diff --git a/router/middleware/settings/doc.go b/router/middleware/settings/doc.go new file mode 100644 index 000000000..2c83d65fc --- /dev/null +++ b/router/middleware/settings/doc.go @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 + +// Package settings provides the ability for inserting +// Vela settings resources into or extracting Vela settings +// resources from the middleware chain for the API. +// +// Usage: +// +// import "github.com/go-vela/server/router/middleware/settings" +package settings diff --git a/router/middleware/allowlist_test.go b/router/middleware/settings_test.go similarity index 60% rename from router/middleware/allowlist_test.go rename to router/middleware/settings_test.go index 2bc59623d..d0b5d2b58 100644 --- a/router/middleware/allowlist_test.go +++ b/router/middleware/settings_test.go @@ -9,12 +9,16 @@ import ( "testing" "github.com/gin-gonic/gin" + + "github.com/go-vela/server/api/types/settings" ) -func TestMiddleware_Allowlist(t *testing.T) { +func TestMiddleware_Settings(t *testing.T) { // setup types - got := []string{""} - want := []string{"foobar"} + want := settings.PlatformMockEmpty() + want.SetCloneImage("target/vela-git") + + got := settings.PlatformMockEmpty() // setup context gin.SetMode(gin.TestMode) @@ -24,9 +28,9 @@ func TestMiddleware_Allowlist(t *testing.T) { context.Request, _ = http.NewRequest(http.MethodGet, "/health", nil) // setup mock server - engine.Use(Allowlist(want)) + engine.Use(Settings(&want)) engine.GET("/health", func(c *gin.Context) { - got = c.Value("allowlist").([]string) + got = *c.Value("settings").(*settings.Platform) c.Status(http.StatusOK) }) @@ -35,10 +39,10 @@ func TestMiddleware_Allowlist(t *testing.T) { engine.ServeHTTP(context.Writer, context.Request) if resp.Code != http.StatusOK { - t.Errorf("Secret returned %v, want %v", resp.Code, http.StatusOK) + t.Errorf("Settings returned %v, want %v", resp.Code, http.StatusOK) } if !reflect.DeepEqual(got, want) { - t.Errorf("Secret is %v, want %v", got, want) + t.Errorf("Settings is %v, want %v", got, want) } } diff --git a/router/middleware/worker_test.go b/router/middleware/worker_test.go index 7717d03c5..5f01dc8a4 100644 --- a/router/middleware/worker_test.go +++ b/router/middleware/worker_test.go @@ -37,10 +37,10 @@ func TestMiddleware_Worker(t *testing.T) { engine.ServeHTTP(context.Writer, context.Request) if resp.Code != http.StatusOK { - t.Errorf("Secret returned %v, want %v", resp.Code, http.StatusOK) + t.Errorf("Worker returned %v, want %v", resp.Code, http.StatusOK) } if !reflect.DeepEqual(got, want) { - t.Errorf("Secret is %v, want %v", got, want) + t.Errorf("Worker is %v, want %v", got, want) } } From 6a9e5098ff3648f58cd2cf9a9538c8e7a5430af4 Mon Sep 17 00:00:00 2001 From: Taylor Mapes Date: Mon, 13 May 2024 09:19:49 -0500 Subject: [PATCH 10/14] chore: Correct the register worker token swagger path (#1128) --- api/admin/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/admin/worker.go b/api/admin/worker.go index e09d0b046..82d7e9b5c 100644 --- a/api/admin/worker.go +++ b/api/admin/worker.go @@ -16,7 +16,7 @@ import ( "github.com/go-vela/types/library" ) -// swagger:operation POST /api/v1/admin/workers/{worker}/register-token admin RegisterToken +// swagger:operation POST /api/v1/admin/workers/{worker}/register admin RegisterToken // // Get a worker registration token // From 93571138ad3f66fc12149b645b0ba78b2efb12d7 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Mon, 13 May 2024 09:34:05 -0500 Subject: [PATCH 11/14] chore: rename settings FromAPI constructor (#1126) Co-authored-by: Easton Crupper <65553218+ecrupper@users.noreply.github.com> --- database/settings/create.go | 2 +- database/settings/update.go | 2 +- database/types/settings.go | 4 ++-- database/types/settings_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/database/settings/create.go b/database/settings/create.go index 0e86f7fbc..7775fb4c9 100644 --- a/database/settings/create.go +++ b/database/settings/create.go @@ -14,7 +14,7 @@ func (e *engine) CreateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Tracef("creating platform settings in the database with %v", s.String()) // cast the api type to database type - settings := types.FromAPI(s) + settings := types.SettingsFromAPI(s) // validate the necessary fields are populated err := settings.Validate() diff --git a/database/settings/update.go b/database/settings/update.go index b72b4d543..c8a3e1679 100644 --- a/database/settings/update.go +++ b/database/settings/update.go @@ -14,7 +14,7 @@ func (e *engine) UpdateSettings(_ context.Context, s *settings.Platform) (*setti e.logger.Trace("updating platform settings in the database") // cast the api type to database type - dbS := types.FromAPI(s) + dbS := types.SettingsFromAPI(s) // validate the necessary fields are populated err := dbS.Validate() diff --git a/database/types/settings.go b/database/types/settings.go index f17bac54b..ea0d47a2a 100644 --- a/database/types/settings.go +++ b/database/types/settings.go @@ -194,9 +194,9 @@ func (ps *Platform) Validate() error { return nil } -// FromAPI converts the API Settings type +// SettingsFromAPI converts the API Settings type // to a database Settings type. -func FromAPI(s *settings.Platform) *Platform { +func SettingsFromAPI(s *settings.Platform) *Platform { settings := &Platform{ ID: sql.NullInt64{Int64: s.GetID(), Valid: true}, Compiler: Compiler{ diff --git a/database/types/settings_test.go b/database/types/settings_test.go index 973e5d88d..40285eb34 100644 --- a/database/types/settings_test.go +++ b/database/types/settings_test.go @@ -166,7 +166,7 @@ func TestTypes_Platform_PlatformFromAPI(t *testing.T) { want := testPlatform() // run test - got := FromAPI(s) + got := SettingsFromAPI(s) if !reflect.DeepEqual(got, want) { t.Errorf("PlatformFromAPI is %v, want %v", got, want) From 927dc715e59f0e14704b0a04117c3d04fb656d68 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Mon, 13 May 2024 16:39:26 -0500 Subject: [PATCH 12/14] fix: platform settings yaml tags (#1129) --- api/types/settings/compiler.go | 6 +++--- api/types/settings/platform.go | 14 +++++++------- api/types/settings/queue.go | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/types/settings/compiler.go b/api/types/settings/compiler.go index a18a31401..147cade94 100644 --- a/api/types/settings/compiler.go +++ b/api/types/settings/compiler.go @@ -5,9 +5,9 @@ package settings import "fmt" type Compiler struct { - CloneImage *string `json:"clone_image,omitempty"` - TemplateDepth *int `json:"template_depth,omitempty"` - StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty"` + CloneImage *string `json:"clone_image,omitempty" yaml:"clone_image,omitempty"` + TemplateDepth *int `json:"template_depth,omitempty" yaml:"template_depth,omitempty"` + StarlarkExecLimit *uint64 `json:"starlark_exec_limit,omitempty" yaml:"starlark_exec_limit,omitempty"` } // GetCloneImage returns the CloneImage field. diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index f72698b66..b494c8faa 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -11,13 +11,13 @@ import ( // swagger:model Platform type Platform struct { ID *int64 `json:"id"` - *Queue `json:"queue"` - *Compiler `json:"compiler"` - RepoAllowlist *[]string `json:"repo_allowlist"` - ScheduleAllowlist *[]string `json:"schedule_allowlist"` - CreatedAt *int64 `json:"created_at,omitempty"` - UpdatedAt *int64 `json:"updated_at,omitempty"` - UpdatedBy *string `json:"updated_by,omitempty"` + *Compiler `json:"compiler,omitempty" yaml:"compiler,omitempty"` + *Queue `json:"queue,omitempty" yaml:"queue,omitempty"` + RepoAllowlist *[]string `json:"repo_allowlist,omitempty" yaml:"repo_allowlist,omitempty"` + ScheduleAllowlist *[]string `json:"schedule_allowlist,omitempty" yaml:"schedule_allowlist,omitempty"` + CreatedAt *int64 `json:"created_at,omitempty" yaml:"created_at,omitempty"` + UpdatedAt *int64 `json:"updated_at,omitempty" yaml:"updated_at,omitempty"` + UpdatedBy *string `json:"updated_by,omitempty" yaml:"updated_by,omitempty"` } // GetID returns the ID field. diff --git a/api/types/settings/queue.go b/api/types/settings/queue.go index 9bf54dfd1..e813b18ae 100644 --- a/api/types/settings/queue.go +++ b/api/types/settings/queue.go @@ -5,7 +5,7 @@ package settings import "fmt" type Queue struct { - Routes *[]string `json:"routes,omitempty"` + Routes *[]string `json:"routes,omitempty" yaml:"routes,omitempty"` } // GetRoutes returns the Routes field. From 4a31c1e4ca8779a8298e488235b5a8a5929d5493 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Thu, 16 May 2024 10:17:32 -0500 Subject: [PATCH 13/14] fix(settings): missing fields in api conversion and func renames (#1130) --- api/admin/settings.go | 15 ++++++++----- api/types/settings/platform.go | 35 ++++++++++++++++++++++------- api/types/settings/platform_test.go | 2 +- cmd/vela-server/server.go | 10 ++------- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/api/admin/settings.go b/api/admin/settings.go index 0b067372b..f567c4263 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -119,7 +119,7 @@ func UpdateSettings(c *gin.Context) { // duplicate settings to not alter the shared pointer _s := new(settings.Platform) - _s.Update(s) + _s.FromSettings(s) // ensure we update the singleton record _s.SetID(1) @@ -254,18 +254,21 @@ func RestoreSettings(c *gin.Context) { return } - s.SetUpdatedAt(time.Now().UTC().Unix()) - s.SetUpdatedBy(u.GetName()) + // initialize a new settings record + _s := settings.FromCLIContext(cliCtx) + + _s.SetUpdatedAt(time.Now().UTC().Unix()) + _s.SetUpdatedBy(u.GetName()) // read in defaults supplied from the cli runtime compilerSettings := compiler.GetSettings() - s.SetCompiler(compilerSettings) + _s.SetCompiler(compilerSettings) queueSettings := queue.GetSettings() - s.SetQueue(queueSettings) + _s.SetQueue(queueSettings) // send API call to update the settings - s, err = database.FromContext(c).UpdateSettings(ctx, s) + s, err = database.FromContext(c).UpdateSettings(ctx, _s) if err != nil { retErr := fmt.Errorf("unable to update (restore) settings: %w", err) diff --git a/api/types/settings/platform.go b/api/types/settings/platform.go index b494c8faa..ff7c58ef3 100644 --- a/api/types/settings/platform.go +++ b/api/types/settings/platform.go @@ -4,6 +4,8 @@ package settings import ( "fmt" + + "github.com/urfave/cli/v2" ) // Platform is the API representation of platform settingps. @@ -20,6 +22,19 @@ type Platform struct { UpdatedBy *string `json:"updated_by,omitempty" yaml:"updated_by,omitempty"` } +// FromCLIContext returns a new Platform record from a cli context. +func FromCLIContext(c *cli.Context) *Platform { + ps := new(Platform) + + // set repos permitted to be added + ps.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) + + // set repos permitted to use schedules + ps.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) + + return ps +} + // GetID returns the ID field. // // When the provided Platform type is nil, or the field within @@ -228,21 +243,25 @@ func (ps *Platform) SetUpdatedBy(v string) { ps.UpdatedBy = &v } -// Update takes another settings record and updates the internal fields, intended -// to be used when the refreshing settings record shared across the server. -func (ps *Platform) Update(newSettingps *Platform) { +// FromSettings takes another settings record and updates the internal fields, +// used when the updating settings and refreshing the record shared across the server. +func (ps *Platform) FromSettings(_ps *Platform) { if ps == nil { return } - if newSettingps == nil { + if _ps == nil { return } - ps.SetCompiler(newSettingps.GetCompiler()) - ps.SetQueue(newSettingps.GetQueue()) - ps.SetRepoAllowlist(newSettingps.GetRepoAllowlist()) - ps.SetScheduleAllowlist(newSettingps.GetScheduleAllowlist()) + ps.SetCompiler(_ps.GetCompiler()) + ps.SetQueue(_ps.GetQueue()) + ps.SetRepoAllowlist(_ps.GetRepoAllowlist()) + ps.SetScheduleAllowlist(_ps.GetScheduleAllowlist()) + + ps.SetCreatedAt(_ps.GetCreatedAt()) + ps.SetUpdatedAt(_ps.GetUpdatedAt()) + ps.SetUpdatedBy(_ps.GetUpdatedBy()) } // String implements the Stringer interface for the Platform type. diff --git a/api/types/settings/platform_test.go b/api/types/settings/platform_test.go index 56c9cec7b..059c7b574 100644 --- a/api/types/settings/platform_test.go +++ b/api/types/settings/platform_test.go @@ -121,7 +121,7 @@ func TestTypes_Platform_Update(t *testing.T) { // run tests for _, test := range tests { - test.platform.Update(test.want) + test.platform.FromSettings(test.want) if diff := cmp.Diff(test.want, test.platform); diff != "" { t.Errorf("(Update: -want +got):\n%s", diff) diff --git a/cmd/vela-server/server.go b/cmd/vela-server/server.go index bae9ec326..dd5b6d6c6 100644 --- a/cmd/vela-server/server.go +++ b/cmd/vela-server/server.go @@ -119,7 +119,7 @@ func server(c *cli.Context) error { logrus.Info("creating initial platform settings") // create initial settings record - ps = new(settings.Platform) + ps = settings.FromCLIContext(c) // singleton record ID should always be 1 ps.SetID(1) @@ -135,12 +135,6 @@ func server(c *cli.Context) error { queueSettings := queue.GetSettings() ps.SetQueue(queueSettings) - // set repos permitted to be added - ps.SetRepoAllowlist(c.StringSlice("vela-repo-allowlist")) - - // set repos permitted to use schedules - ps.SetScheduleAllowlist(c.StringSlice("vela-schedule-allowlist")) - // create the settings record in the database _, err = database.CreateSettings(context.Background(), ps) if err != nil { @@ -252,7 +246,7 @@ func server(c *cli.Context) error { } // update the internal fields for the shared settings record - ps.Update(newSettings) + ps.FromSettings(newSettings) } }) From 178e678a82130df48e2bd1c5a86050e35972a12b Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Thu, 16 May 2024 10:52:23 -0500 Subject: [PATCH 14/14] fix(settings): swagger missing 401 responses (#1132) --- api/admin/settings.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/api/admin/settings.go b/api/admin/settings.go index f567c4263..f04109680 100644 --- a/api/admin/settings.go +++ b/api/admin/settings.go @@ -36,6 +36,10 @@ import ( // type: json // schema: // "$ref": "#/definitions/Platform" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" // '404': // description: Unable to retrieve settings // schema: @@ -88,6 +92,10 @@ func GetSettings(c *gin.Context) { // description: Unable to update settings — bad request // schema: // "$ref": "#/definitions/Error" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" // '404': // description: Unable to retrieve platform settings to update // schema: @@ -206,6 +214,10 @@ func UpdateSettings(c *gin.Context) { // type: json // schema: // "$ref": "#/definitions/Platform" +// '401': +// description: Unauthorized +// schema: +// "$ref": "#/definitions/Error" // '404': // description: Unable to retrieve settings to restore // schema: