From 336da1e98be0a67f4bde4473b4bb840ff478eff0 Mon Sep 17 00:00:00 2001 From: ecrupper Date: Wed, 1 Nov 2023 12:59:20 -0500 Subject: [PATCH] refactor(types): move worker type to server and nest API object --- api/types/worker.go | 370 +++++++++++++++++++++++ api/types/worker_test.go | 224 ++++++++++++++ api/worker/create.go | 3 +- api/worker/get.go | 17 +- api/worker/list.go | 37 ++- api/worker/update.go | 8 +- database/integration_test.go | 13 +- database/types/sanitize.go | 42 +++ database/types/sanitize_test.go | 83 +++++ database/types/worker.go | 191 ++++++++++++ database/types/worker_test.go | 216 +++++++++++++ database/worker/create.go | 10 +- database/worker/create_test.go | 2 +- database/worker/delete.go | 8 +- database/worker/get.go | 10 +- database/worker/get_hostname.go | 10 +- database/worker/get_hostname_test.go | 5 +- database/worker/get_test.go | 5 +- database/worker/interface.go | 14 +- database/worker/list.go | 12 +- database/worker/list_test.go | 10 +- database/worker/update.go | 10 +- database/worker/update_test.go | 2 +- database/worker/worker.go | 22 ++ database/worker/worker_test.go | 52 +++- go.mod | 4 +- mock/server/worker.go | 67 +++- router/middleware/logger_test.go | 3 +- router/middleware/worker/context.go | 8 +- router/middleware/worker/context_test.go | 6 +- router/middleware/worker/worker.go | 4 +- router/middleware/worker/worker_test.go | 12 +- 32 files changed, 1393 insertions(+), 87 deletions(-) create mode 100644 api/types/worker.go create mode 100644 api/types/worker_test.go create mode 100644 database/types/sanitize.go create mode 100644 database/types/sanitize_test.go create mode 100644 database/types/worker.go create mode 100644 database/types/worker_test.go diff --git a/api/types/worker.go b/api/types/worker.go new file mode 100644 index 000000000..78f86cf1c --- /dev/null +++ b/api/types/worker.go @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + + "github.com/go-vela/types/library" +) + +// Worker is the library representation of a worker. +// +// swagger:model Worker +type Worker struct { + ID *int64 `json:"id,omitempty"` + Hostname *string `json:"hostname,omitempty"` + Address *string `json:"address,omitempty"` + Routes *[]string `json:"routes,omitempty"` + Active *bool `json:"active,omitempty"` + Status *string `json:"status,omitempty"` + LastStatusUpdateAt *int64 `json:"last_status_update_at,omitempty"` + RunningBuilds []*library.Build `json:"running_builds,omitempty"` + LastBuildStartedAt *int64 `json:"last_build_started_at,omitempty"` + LastBuildFinishedAt *int64 `json:"last_build_finished_at,omitempty"` + LastCheckedIn *int64 `json:"last_checked_in,omitempty"` + BuildLimit *int64 `json:"build_limit,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetID() int64 { + // return zero value if Worker type or ID field is nil + if w == nil || w.ID == nil { + return 0 + } + + return *w.ID +} + +// GetHostname returns the Hostname field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetHostname() string { + // return zero value if Worker type or Hostname field is nil + if w == nil || w.Hostname == nil { + return "" + } + + return *w.Hostname +} + +// GetAddress returns the Address field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetAddress() string { + // return zero value if Worker type or Address field is nil + if w == nil || w.Address == nil { + return "" + } + + return *w.Address +} + +// GetRoutes returns the Routes field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRoutes() []string { + // return zero value if Worker type or Routes field is nil + if w == nil || w.Routes == nil { + return []string{} + } + + return *w.Routes +} + +// GetActive returns the Active field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetActive() bool { + // return zero value if Worker type or Active field is nil + if w == nil || w.Active == nil { + return false + } + + return *w.Active +} + +// GetStatus returns the Status field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetStatus() string { + // return zero value if Worker type or Status field is nil + if w == nil || w.Status == nil { + return "" + } + + return *w.Status +} + +// GetLastStatusUpdateAt returns the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastStatusUpdateAt() int64 { + // return zero value if Worker type or LastStatusUpdateAt field is nil + if w == nil || w.LastStatusUpdateAt == nil { + return 0 + } + + return *w.LastStatusUpdateAt +} + +// GetRunningBuilds returns the RunningBuilds field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRunningBuilds() []*library.Build { + // return zero value if Worker type or RunningBuilds field is nil + if w == nil || w.RunningBuilds == nil { + return []*library.Build{} + } + + return w.RunningBuilds +} + +// GetLastBuildStartedAt returns the LastBuildStartedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildStartedAt() int64 { + // return zero value if Worker type or LastBuildStartedAt field is nil + if w == nil || w.LastBuildStartedAt == nil { + return 0 + } + + return *w.LastBuildStartedAt +} + +// GetLastBuildFinishedAt returns the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildFinishedAt() int64 { + // return zero value if Worker type or LastBuildFinishedAt field is nil + if w == nil || w.LastBuildFinishedAt == nil { + return 0 + } + + return *w.LastBuildFinishedAt +} + +// GetLastCheckedIn returns the LastCheckedIn field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastCheckedIn() int64 { + // return zero value if Worker type or LastCheckedIn field is nil + if w == nil || w.LastCheckedIn == nil { + return 0 + } + + return *w.LastCheckedIn +} + +// GetBuildLimit returns the BuildLimit field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetBuildLimit() int64 { + // return zero value if Worker type or BuildLimit field is nil + if w == nil || w.BuildLimit == nil { + return 0 + } + + return *w.BuildLimit +} + +// SetID sets the ID field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetID(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.ID = &v +} + +// SetHostname sets the Hostname field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetHostname(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Hostname = &v +} + +// SetAddress sets the Address field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetAddress(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Address = &v +} + +// SetRoutes sets the Routes field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRoutes(v []string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Routes = &v +} + +// SetActive sets the Active field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetActive(v bool) { + // return if Worker type is nil + if w == nil { + return + } + + w.Active = &v +} + +// SetStatus sets the Status field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetStatus(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Status = &v +} + +// SetLastStatusUpdateAt sets the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastStatusUpdateAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastStatusUpdateAt = &v +} + +// SetRunningBuilds sets the RunningBuilds field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRunningBuilds(builds []*library.Build) { + // return if Worker type is nil + if w == nil { + return + } + + w.RunningBuilds = builds +} + +// SetLastBuildStartedAt sets the LastBuildStartedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildStartedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildStartedAt = &v +} + +// SetLastBuildFinishedAt sets the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildFinishedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildFinishedAt = &v +} + +// SetLastCheckedIn sets the LastCheckedIn field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastCheckedIn(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastCheckedIn = &v +} + +// SetBuildLimit sets the LastBuildLimit field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetBuildLimit(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.BuildLimit = &v +} + +// String implements the Stringer interface for the Worker type. +func (w *Worker) String() string { + return fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) +} diff --git a/api/types/worker_test.go b/api/types/worker_test.go new file mode 100644 index 000000000..19d2defb5 --- /dev/null +++ b/api/types/worker_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/go-vela/types/library" +) + +func TestTypes_Worker_Getters(t *testing.T) { + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: new(Worker), + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + if test.worker.GetID() != test.want.GetID() { + t.Errorf("GetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("GetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("Getaddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("GetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("GetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("GetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("GetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if !reflect.DeepEqual(test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) { + t.Errorf("GetRunningBuildIDs is %v, want %v", test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("GetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("GetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("GetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("GetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_Setters(t *testing.T) { + // setup types + var w *Worker + + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: w, + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + test.worker.SetID(test.want.GetID()) + test.worker.SetHostname(test.want.GetHostname()) + test.worker.SetAddress(test.want.GetAddress()) + test.worker.SetRoutes(test.want.GetRoutes()) + test.worker.SetActive(test.want.GetActive()) + test.worker.SetStatus(test.want.GetStatus()) + test.worker.SetLastStatusUpdateAt(test.want.GetLastStatusUpdateAt()) + test.worker.SetRunningBuilds(test.want.GetRunningBuilds()) + test.worker.SetLastBuildStartedAt(test.want.GetLastBuildStartedAt()) + test.worker.SetLastBuildFinishedAt(test.want.GetLastBuildFinishedAt()) + test.worker.SetLastCheckedIn(test.want.GetLastCheckedIn()) + test.worker.SetBuildLimit(test.want.GetBuildLimit()) + + if test.worker.GetID() != test.want.GetID() { + t.Errorf("SetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("SetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("SetAddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("SetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("SetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("SetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("SetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("SetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("SetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("SetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("SetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_String(t *testing.T) { + // setup types + w := testWorker() + + want := fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) + + // run test + got := w.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testWorker is a test helper function to create a Worker +// type with all fields set to a fake value. +func testWorker() *Worker { + b := new(library.Build) + b.SetID(1) + + w := new(Worker) + + w.SetID(1) + w.SetHostname("worker_0") + w.SetAddress("http://localhost:8080") + w.SetRoutes([]string{"vela"}) + w.SetActive(true) + w.SetStatus("available") + w.SetLastStatusUpdateAt(time.Time{}.UTC().Unix()) + w.SetRunningBuilds([]*library.Build{b}) + w.SetLastBuildStartedAt(time.Time{}.UTC().Unix()) + w.SetLastBuildFinishedAt(time.Time{}.UTC().Unix()) + w.SetLastCheckedIn(time.Time{}.UTC().Unix()) + w.SetBuildLimit(2) + + return w +} diff --git a/api/worker/create.go b/api/worker/create.go index 07edf3acc..d91898885 100644 --- a/api/worker/create.go +++ b/api/worker/create.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/claims" @@ -58,7 +59,7 @@ func CreateWorker(c *gin.Context) { ctx := c.Request.Context() // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { diff --git a/api/worker/get.go b/api/worker/get.go index a8a4d6931..99cb17204 100644 --- a/api/worker/get.go +++ b/api/worker/get.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -55,14 +56,20 @@ func GetWorker(c *gin.Context) { "worker": w.GetHostname(), }).Infof("reading worker %s", w.GetHostname()) - w, err := database.FromContext(c).GetWorkerForHostname(ctx, w.GetHostname()) - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) + var rBs []*library.Build + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) - util.HandleError(c, http.StatusNotFound, retErr) + return + } - return + rBs = append(rBs, build) } + w.SetRunningBuilds(rBs) + c.JSON(http.StatusOK, w) } diff --git a/api/worker/list.go b/api/worker/list.go index 6b24484a6..1bbd9bbc5 100644 --- a/api/worker/list.go +++ b/api/worker/list.go @@ -10,6 +10,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -22,6 +23,12 @@ import ( // - application/json // security: // - ApiKeyAuth: [] +// parameters: +// - in: query +// name: links +// description: Include links to builds currently running +// type: boolean +// default: false // responses: // '200': // description: Successfully retrieved the list of workers @@ -48,7 +55,15 @@ func ListWorkers(c *gin.Context) { "user": u.GetName(), }).Info("reading workers") - w, err := database.FromContext(c).ListWorkers(ctx) + var filters = map[string]interface{}{} + + active := c.Query("active") + + if len(active) > 0 { + filters["active"] = active + } + + workers, err := database.FromContext(c).ListWorkers(ctx) if err != nil { retErr := fmt.Errorf("unable to get workers: %w", err) @@ -57,5 +72,23 @@ func ListWorkers(c *gin.Context) { return } - c.JSON(http.StatusOK, w) + for _, w := range workers { + var rBs []*library.Build + + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + rBs = append(rBs, build) + } + + w.SetRunningBuilds(rBs) + } + + c.JSON(http.StatusOK, workers) } diff --git a/api/worker/update.go b/api/worker/update.go index d95ddf7af..e5b842a0d 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -7,11 +7,11 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -71,7 +71,7 @@ func UpdateWorker(c *gin.Context) { }).Infof("updating worker %s", w.GetHostname()) // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { @@ -97,9 +97,9 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } - if input.RunningBuildIDs != nil { + if input.RunningBuilds != nil { // update runningBuildIDs if set - w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + w.SetRunningBuilds(input.GetRunningBuilds()) } if len(input.GetStatus()) > 0 { diff --git a/database/integration_test.go b/database/integration_test.go index b30176004..0cc6b1659 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/executable" "github.com/go-vela/server/database/hook" @@ -42,7 +43,7 @@ type Resources struct { Services []*library.Service Steps []*library.Step Users []*library.User - Workers []*library.Worker + Workers []*api.Worker } func TestDatabase_Integration(t *testing.T) { @@ -2313,7 +2314,7 @@ func newResources() *Resources { userTwo.SetActive(true) userTwo.SetAdmin(false) - workerOne := new(library.Worker) + workerOne := new(api.Worker) workerOne.SetID(1) workerOne.SetHostname("worker-1.example.com") workerOne.SetAddress("https://worker-1.example.com") @@ -2321,13 +2322,13 @@ func newResources() *Resources { workerOne.SetActive(true) workerOne.SetStatus("available") workerOne.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerOne.SetRunningBuildIDs([]string{"12345"}) + workerOne.SetRunningBuilds([]*library.Build{buildOne}) workerOne.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerOne.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerOne.SetLastCheckedIn(time.Now().UTC().Unix()) workerOne.SetBuildLimit(1) - workerTwo := new(library.Worker) + workerTwo := new(api.Worker) workerTwo.SetID(2) workerTwo.SetHostname("worker-2.example.com") workerTwo.SetAddress("https://worker-2.example.com") @@ -2335,7 +2336,7 @@ func newResources() *Resources { workerTwo.SetActive(true) workerTwo.SetStatus("available") workerTwo.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerTwo.SetRunningBuildIDs([]string{"12345"}) + workerTwo.SetRunningBuilds([]*library.Build{buildTwo}) workerTwo.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerTwo.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerTwo.SetLastCheckedIn(time.Now().UTC().Unix()) @@ -2354,7 +2355,7 @@ func newResources() *Resources { Services: []*library.Service{serviceOne, serviceTwo}, Steps: []*library.Step{stepOne, stepTwo}, Users: []*library.User{userOne, userTwo}, - Workers: []*library.Worker{workerOne, workerTwo}, + Workers: []*api.Worker{workerOne, workerTwo}, } } diff --git a/database/types/sanitize.go b/database/types/sanitize.go new file mode 100644 index 000000000..c21918194 --- /dev/null +++ b/database/types/sanitize.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "html" + "net/url" + "strings" + + "github.com/microcosm-cc/bluemonday" +) + +// sanitize is a helper function to verify the provided input +// field does not contain HTML content. If the input field +// does contain HTML, then the function will sanitize and +// potentially remove the HTML if deemed malicious. +func sanitize(field string) string { + // create new HTML input microcosm-cc/bluemonday policy + p := bluemonday.StrictPolicy() + + // create a URL query unescaped string from the field + queryUnescaped, err := url.QueryUnescape(field) + if err != nil { + // overwrite URL query unescaped string with field + queryUnescaped = field + } + + // create an HTML escaped string from the field + htmlEscaped := html.EscapeString(queryUnescaped) + + // create a microcosm-cc/bluemonday escaped string from the field + bluemondayEscaped := p.Sanitize(queryUnescaped) + + // check if the field contains html + if !strings.EqualFold(htmlEscaped, bluemondayEscaped) { + // create new HTML input microcosm-cc/bluemonday policy + return bluemondayEscaped + } + + // return the unmodified field + return field +} diff --git a/database/types/sanitize_test.go b/database/types/sanitize_test.go new file mode 100644 index 000000000..a886e982a --- /dev/null +++ b/database/types/sanitize_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "testing" +) + +func TestTypes_Sanitize(t *testing.T) { + // setup tests + tests := []struct { + name string + value string + want string + }{ + { + name: "percent", + value: `%`, + want: `%`, + }, + { + name: "quoted", + value: `"hello"`, + want: `"hello"`, + }, + { + name: "email", + value: `OctoKitty@github.com`, + want: `OctoKitty@github.com`, + }, + { + name: "url", + value: `https://github.com/go-vela`, + want: `https://github.com/go-vela`, + }, + { + name: "encoded", + value: `+ added foo %25 + updated bar %22 +`, + want: `+ added foo %25 + updated bar %22 +`, + }, + { + name: "html with headers", + value: `Merge pull request #1 from me/patch-1\n\n

hello

is now

hello

`, + want: `Merge pull request #1 from me/patch-1\n\nhello is now hello`, + }, + { + name: "html with email", + value: `Co-authored-by: OctoKitty `, + want: `Co-authored-by: OctoKitty `, + }, + { + name: "html with href", + value: `Google`, + want: `Google`, + }, + { + name: "local cross-site script", + value: ``, + want: ``, + }, + { + name: "remote cross-site script", + value: ``, + want: ``, + }, + { + name: "embedded cross-site script", + value: `%3cDIV%20STYLE%3d%22width%3a%20expression(alert('XSS'))%3b%22%3e`, + want: ``, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := sanitize(test.value) + + if got != test.want { + t.Errorf("sanitize is %v, want %v", got, test.want) + } + }) + } +} diff --git a/database/types/worker.go b/database/types/worker.go new file mode 100644 index 000000000..075ce916b --- /dev/null +++ b/database/types/worker.go @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "errors" + "fmt" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/lib/pq" +) + +var ( + // ErrEmptyWorkerHost defines the error type when a + // Worker type has an empty Host field provided. + ErrEmptyWorkerHost = errors.New("empty worker address provided") + + // ErrEmptyWorkerAddress defines the error type when a + // Worker type has an empty Address field provided. + ErrEmptyWorkerAddress = errors.New("empty worker address provided") + + // ErrExceededRunningBuildIDsLimit defines the error type when a + // Worker type has RunningBuildIDs field provided that exceeds the database limit. + ErrExceededRunningBuildIDsLimit = errors.New("exceeded running build ids limit") +) + +// Worker is the database representation of a worker. +type Worker struct { + ID sql.NullInt64 `sql:"id"` + Hostname sql.NullString `sql:"hostname"` + Address sql.NullString `sql:"address"` + Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` + Active sql.NullBool `sql:"active"` + Status sql.NullString `sql:"status"` + LastStatusUpdateAt sql.NullInt64 `sql:"last_status_update_at"` + RunningBuildIDs pq.StringArray `sql:"running_build_ids" gorm:"type:varchar(500)"` + LastBuildStartedAt sql.NullInt64 `sql:"last_build_started_at"` + LastBuildFinishedAt sql.NullInt64 `sql:"last_build_finished_at"` + LastCheckedIn sql.NullInt64 `sql:"last_checked_in"` + BuildLimit sql.NullInt64 `sql:"build_limit"` +} + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Build type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (w *Worker) Nullify() *Worker { + if w == nil { + return nil + } + + // check if the ID field should be false + if w.ID.Int64 == 0 { + w.ID.Valid = false + } + + // check if the Hostname field should be false + if len(w.Hostname.String) == 0 { + w.Hostname.Valid = false + } + + // check if the Address field should be false + if len(w.Address.String) == 0 { + w.Address.Valid = false + } + + // check if the Status field should be false + if len(w.Status.String) == 0 { + w.Status.Valid = false + } + + // check if the LastStatusUpdateAt field should be false + if w.LastStatusUpdateAt.Int64 == 0 { + w.LastStatusUpdateAt.Valid = false + } + + // check if the LastBuildStartedAt field should be false + if w.LastBuildStartedAt.Int64 == 0 { + w.LastBuildStartedAt.Valid = false + } + + // check if the LastBuildFinishedAt field should be false + if w.LastBuildFinishedAt.Int64 == 0 { + w.LastBuildFinishedAt.Valid = false + } + + // check if the LastCheckedIn field should be false + if w.LastCheckedIn.Int64 == 0 { + w.LastCheckedIn.Valid = false + } + + if w.BuildLimit.Int64 == 0 { + w.BuildLimit.Valid = false + } + + return w +} + +// ToAPI converts the Worker type +// to an API Worker type. +func (w *Worker) ToAPI(builds []*library.Build) *types.Worker { + worker := new(types.Worker) + + worker.SetID(w.ID.Int64) + worker.SetHostname(w.Hostname.String) + worker.SetAddress(w.Address.String) + worker.SetRoutes(w.Routes) + worker.SetActive(w.Active.Bool) + worker.SetStatus(w.Status.String) + worker.SetLastStatusUpdateAt(w.LastStatusUpdateAt.Int64) + worker.SetRunningBuilds(builds) + worker.SetLastBuildStartedAt(w.LastBuildStartedAt.Int64) + worker.SetLastBuildFinishedAt(w.LastBuildFinishedAt.Int64) + worker.SetLastCheckedIn(w.LastCheckedIn.Int64) + worker.SetBuildLimit(w.BuildLimit.Int64) + + return worker +} + +// Validate verifies the necessary fields for +// the Worker type are populated correctly. +func (w *Worker) Validate() error { + // verify the Host field is populated + if len(w.Hostname.String) == 0 { + return ErrEmptyWorkerHost + } + + // verify the Address field is populated + if len(w.Address.String) == 0 { + return ErrEmptyWorkerAddress + } + + // calculate total size of RunningBuildIds + total := 0 + for _, f := range w.RunningBuildIDs { + total += len(f) + } + + // verify the RunningBuildIds field is within the database constraints + // len is to factor in number of comma separators included in the database field, + // removing 1 due to the last item not having an appended comma + if (total + len(w.RunningBuildIDs) - 1) > constants.RunningBuildIDsMaxSize { + return ErrExceededRunningBuildIDsLimit + } + + // ensure that all Worker string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + w.Hostname = sql.NullString{String: sanitize(w.Hostname.String), Valid: w.Hostname.Valid} + w.Address = sql.NullString{String: sanitize(w.Address.String), Valid: w.Address.Valid} + + // ensure that all Routes are sanitized + // to avoid unsafe HTML content + for i, v := range w.Routes { + w.Routes[i] = sanitize(v) + } + + return nil +} + +// WorkerFromAPI converts the library worker type +// to a database worker type. +func WorkerFromAPI(w *types.Worker) *Worker { + var rBs []string + + for _, b := range w.GetRunningBuilds() { + rBs = append(rBs, fmt.Sprint(b.GetID())) + } + + worker := &Worker{ + ID: sql.NullInt64{Int64: w.GetID(), Valid: true}, + Hostname: sql.NullString{String: w.GetHostname(), Valid: true}, + Address: sql.NullString{String: w.GetAddress(), Valid: true}, + Routes: pq.StringArray(w.GetRoutes()), + Active: sql.NullBool{Bool: w.GetActive(), Valid: true}, + Status: sql.NullString{String: w.GetStatus(), Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: w.GetLastStatusUpdateAt(), Valid: true}, + RunningBuildIDs: pq.StringArray(rBs), + LastBuildStartedAt: sql.NullInt64{Int64: w.GetLastBuildStartedAt(), Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: w.GetLastBuildFinishedAt(), Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: w.GetLastCheckedIn(), Valid: true}, + BuildLimit: sql.NullInt64{Int64: w.GetBuildLimit(), Valid: true}, + } + + return worker.Nullify() +} diff --git a/database/types/worker_test.go b/database/types/worker_test.go new file mode 100644 index 000000000..5a7a65999 --- /dev/null +++ b/database/types/worker_test.go @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "database/sql" + "reflect" + "strconv" + "testing" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/types/library" +) + +func TestTypes_Worker_Nullify(t *testing.T) { + // setup types + var w *Worker + + want := &Worker{ + ID: sql.NullInt64{Int64: 0, Valid: false}, + Hostname: sql.NullString{String: "", Valid: false}, + Address: sql.NullString{String: "", Valid: false}, + Active: sql.NullBool{Bool: false, Valid: false}, + Status: sql.NullString{String: "", Valid: false}, + LastStatusUpdateAt: sql.NullInt64{Int64: 0, Valid: false}, + LastBuildStartedAt: sql.NullInt64{Int64: 0, Valid: false}, + LastBuildFinishedAt: sql.NullInt64{Int64: 0, Valid: false}, + LastCheckedIn: sql.NullInt64{Int64: 0, Valid: false}, + BuildLimit: sql.NullInt64{Int64: 0, Valid: false}, + } + + // setup tests + tests := []struct { + repo *Worker + want *Worker + }{ + { + repo: testWorker(), + want: testWorker(), + }, + { + repo: w, + want: nil, + }, + { + repo: new(Worker), + want: want, + }, + } + + // run tests + for _, test := range tests { + got := test.repo.Nullify() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("Nullify is %v, want %v", got, test.want) + } + } +} + +func TestTypes_Worker_ToAPI(t *testing.T) { + // setup types + b := new(library.Build) + b.SetID(12345) + + builds := []*library.Build{b} + + want := new(types.Worker) + + want.SetID(1) + want.SetHostname("worker_0") + want.SetAddress("http://localhost:8080") + want.SetRoutes([]string{"vela"}) + want.SetActive(true) + want.SetStatus("available") + want.SetLastStatusUpdateAt(1563474077) + want.SetRunningBuilds(builds) + want.SetLastBuildStartedAt(1563474077) + want.SetLastBuildFinishedAt(1563474077) + want.SetLastCheckedIn(1563474077) + want.SetBuildLimit(2) + + // run test + got := testWorker().ToAPI(builds) + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestTypes_Worker_Validate(t *testing.T) { + // setup tests + tests := []struct { + failure bool + worker *Worker + }{ + { + failure: false, + worker: testWorker(), + }, + { // no Hostname set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + { // no Address set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + { // invalid RunningBuildIDs set for worker + failure: true, + worker: &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Active: sql.NullBool{Bool: true, Valid: true}, + RunningBuildIDs: exceededRunningBuildIDs(), + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + }, + }, + } + + // run tests + for _, test := range tests { + err := test.worker.Validate() + + if test.failure { + if err == nil { + t.Errorf("Validate should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("Validate returned err: %v", err) + } + } +} + +func TestTypes_WorkerFromLibrary(t *testing.T) { + // setup types + b := new(library.Build) + b.SetID(12345) + + builds := []*library.Build{b} + + w := new(types.Worker) + + w.SetID(1) + w.SetHostname("worker_0") + w.SetAddress("http://localhost:8080") + w.SetRoutes([]string{"vela"}) + w.SetActive(true) + w.SetStatus("available") + w.SetLastStatusUpdateAt(1563474077) + w.SetRunningBuilds(builds) + w.SetLastBuildStartedAt(1563474077) + w.SetLastBuildFinishedAt(1563474077) + w.SetLastCheckedIn(1563474077) + w.SetBuildLimit(2) + + want := testWorker() + + // run test + got := WorkerFromAPI(w) + + if !reflect.DeepEqual(got, want) { + t.Errorf("WorkerFromLibrary is %v, want %v", got, want) + } +} + +// testWorker is a test helper function to create a Worker +// type with all fields set to a fake value. +func testWorker() *Worker { + return &Worker{ + ID: sql.NullInt64{Int64: 1, Valid: true}, + Hostname: sql.NullString{String: "worker_0", Valid: true}, + Address: sql.NullString{String: "http://localhost:8080", Valid: true}, + Routes: []string{"vela"}, + Active: sql.NullBool{Bool: true, Valid: true}, + Status: sql.NullString{String: "available", Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + RunningBuildIDs: []string{"12345"}, + LastBuildStartedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: 1563474077, Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: 1563474077, Valid: true}, + BuildLimit: sql.NullInt64{Int64: 2, Valid: true}, + } +} + +// exceededRunningBuildIDs returns a list of valid running builds that exceed the maximum size. +func exceededRunningBuildIDs() []string { + // initialize empty runningBuildIDs + runningBuildIDs := []string{} + + // add enough build ids to exceed the character limit + for i := 0; i < 50; i++ { + // construct runningBuildID + // use i to adhere to unique runningBuildIDs + runningBuildID := "1234567890-" + strconv.Itoa(i) + + runningBuildIDs = append(runningBuildIDs, runningBuildID) + } + + return runningBuildIDs +} diff --git a/database/worker/create.go b/database/worker/create.go index 12b386319..7818bd2db 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // CreateWorker creates a new worker in the database. -func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("creating worker %s in the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // validate the necessary fields are populated // @@ -33,5 +33,5 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Create(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/create_test.go b/database/worker/create_test.go index dad81473a..6809bb33e 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -28,7 +28,7 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "workers" ("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/delete.go b/database/worker/delete.go index ba846db91..a2eb6f566 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // DeleteWorker deletes an existing worker from the database. -func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { +func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("deleting worker %s from the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // send query to the database return e.client. diff --git a/database/worker/get.go b/database/worker/get.go index 12201d4d5..7aca60b76 100644 --- a/database/worker/get.go +++ b/database/worker/get.go @@ -5,17 +5,17 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // GetWorker gets a worker by ID from the database. -func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, error) { +func (e *engine) GetWorker(ctx context.Context, id int64) (*api.Worker, error) { e.logger.Tracef("getting worker %d from the database", id) // variable to store query results - w := new(database.Worker) + w := new(types.Worker) // send query to the database and store result in variable err := e.client. @@ -30,5 +30,5 @@ func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, erro // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go index 3370620a0..4b4b54ad2 100644 --- a/database/worker/get_hostname.go +++ b/database/worker/get_hostname.go @@ -5,20 +5,20 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // GetWorkerForHostname gets a worker by hostname from the database. -func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*library.Worker, error) { +func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": hostname, }).Tracef("getting worker %s from the database", hostname) // variable to store query results - w := new(database.Worker) + w := new(types.Worker) // send query to the database and store result in variable err := e.client. @@ -33,5 +33,5 @@ func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*li // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go index fc4802d70..87e9e1e0c 100644 --- a/database/worker/get_hostname_test.go +++ b/database/worker/get_hostname_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorkerForName(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) // sqlmock cannot parse string array values _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/get_test.go b/database/worker/get_test.go index e5249fda1..a5f6f56d7 100644 --- a/database/worker/get_test.go +++ b/database/worker/get_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorker(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/interface.go b/database/worker/interface.go index c594c556a..fbb2433ef 100644 --- a/database/worker/interface.go +++ b/database/worker/interface.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + "github.com/go-vela/server/api/types" ) // WorkerInterface represents the Vela interface for worker @@ -29,15 +29,15 @@ type WorkerInterface interface { // CountWorkers defines a function that gets the count of all workers. CountWorkers(context.Context) (int64, error) // CreateWorker defines a function that creates a new worker. - CreateWorker(context.Context, *library.Worker) (*library.Worker, error) + CreateWorker(context.Context, *types.Worker) (*types.Worker, error) // DeleteWorker defines a function that deletes an existing worker. - DeleteWorker(context.Context, *library.Worker) error + DeleteWorker(context.Context, *types.Worker) error // GetWorker defines a function that gets a worker by ID. - GetWorker(context.Context, int64) (*library.Worker, error) + GetWorker(context.Context, int64) (*types.Worker, error) // GetWorkerForHostname defines a function that gets a worker by hostname. - GetWorkerForHostname(context.Context, string) (*library.Worker, error) + GetWorkerForHostname(context.Context, string) (*types.Worker, error) // ListWorkers defines a function that gets a list of all workers. - ListWorkers(context.Context) ([]*library.Worker, error) + ListWorkers(context.Context) ([]*types.Worker, error) // UpdateWorker defines a function that updates an existing worker. - UpdateWorker(context.Context, *library.Worker) (*library.Worker, error) + UpdateWorker(context.Context, *types.Worker) (*types.Worker, error) } diff --git a/database/worker/list.go b/database/worker/list.go index dbf8ff2a7..23910504b 100644 --- a/database/worker/list.go +++ b/database/worker/list.go @@ -5,19 +5,19 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListWorkers gets a list of all workers from the database. -func (e *engine) ListWorkers(ctx context.Context) ([]*library.Worker, error) { +func (e *engine) ListWorkers(ctx context.Context) ([]*api.Worker, error) { e.logger.Trace("listing all workers from the database") // variables to store query results and return value count := int64(0) - w := new([]database.Worker) - workers := []*library.Worker{} + w := new([]types.Worker) + workers := []*api.Worker{} // count the results count, err := e.CountWorkers(ctx) @@ -47,7 +47,7 @@ func (e *engine) ListWorkers(ctx context.Context) ([]*library.Worker, error) { // convert query result to library type // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - workers = append(workers, tmp.ToLibrary()) + workers = append(workers, tmp.ToAPI(convertToBuilds(tmp.RunningBuildIDs))) } return workers, nil diff --git a/database/worker/list_test.go b/database/worker/list_test.go index 9c5331fe4..0786d8ebf 100644 --- a/database/worker/list_test.go +++ b/database/worker/list_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_ListWorkers(t *testing.T) { @@ -18,12 +18,14 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { _workerOne.SetHostname("worker_0") _workerOne.SetAddress("localhost") _workerOne.SetActive(true) + _workerOne.SetRunningBuilds(nil) _workerTwo := testWorker() _workerTwo.SetID(2) _workerTwo.SetHostname("worker_1") _workerTwo.SetAddress("localhost") _workerTwo.SetActive(true) + _workerTwo.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -61,19 +63,19 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { failure bool name string database *engine - want []*library.Worker + want []*api.Worker }{ { failure: false, name: "postgres", database: _postgres, - want: []*library.Worker{_workerOne, _workerTwo}, + want: []*api.Worker{_workerOne, _workerTwo}, }, { failure: false, name: "sqlite3", database: _sqlite, - want: []*library.Worker{_workerOne, _workerTwo}, + want: []*api.Worker{_workerOne, _workerTwo}, }, } diff --git a/database/worker/update.go b/database/worker/update.go index 3cae0d164..c162af9f9 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -5,14 +5,14 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/database/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // UpdateWorker updates an existing worker in the database. -func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("updating worker %s in the database", w.GetHostname()) @@ -20,7 +20,7 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := types.WorkerFromAPI(w) // validate the necessary fields are populated // @@ -33,5 +33,5 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Save(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/update_test.go b/database/worker/update_test.go index d077bc567..5a483660b 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -25,7 +25,7 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { _mock.ExpectExec(`UPDATE "workers" SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11 WHERE "id" = $12`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker.go b/database/worker/worker.go index 09b69176a..aaeed7f10 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -5,8 +5,10 @@ package worker import ( "context" "fmt" + "strconv" "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" "gorm.io/gorm" @@ -79,3 +81,23 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } + +// convertToBuilds is a helper function that generates build objects with ID fields given a list of IDs. +func convertToBuilds(ids []string) []*library.Build { + // create stripped build objects holding the IDs + var rBs []*library.Build + + for _, b := range ids { + id, err := strconv.ParseInt(b, 10, 64) + if err != nil { + return nil + } + + build := new(library.Build) + build.SetID(id) + + rBs = append(rBs, build) + } + + return rBs +} diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index a219d1ffc..a7f47428d 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" @@ -106,6 +107,48 @@ func TestWorker_New(t *testing.T) { } } +func TestWorker_convertToBuilds(t *testing.T) { + _buildOne := new(library.Build) + _buildOne.SetID(1) + + _buildTwo := new(library.Build) + _buildTwo.SetID(2) + + // setup tests + tests := []struct { + name string + ids []string + want []*library.Build + }{ + { + name: "one id", + ids: []string{"1"}, + want: []*library.Build{_buildOne}, + }, + { + name: "multiple ids", + ids: []string{"1", "2"}, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + name: "not int64", + ids: []string{"1", "foo"}, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := convertToBuilds(test.ids) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("convertToBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + // testPostgres is a helper function to create a Postgres engine for testing. func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { // create the new mock sql database @@ -166,8 +209,11 @@ func testSqlite(t *testing.T) *engine { // testWorker is a test helper function to create a library // Worker type with all fields set to their zero values. -func testWorker() *library.Worker { - return &library.Worker{ +func testWorker() *api.Worker { + b := new(library.Build) + b.SetID(1) + + return &api.Worker{ ID: new(int64), Hostname: new(string), Address: new(string), @@ -175,7 +221,7 @@ func testWorker() *library.Worker { Active: new(bool), Status: new(string), LastStatusUpdateAt: new(int64), - RunningBuildIDs: new([]string), + RunningBuilds: []*library.Build{b}, LastBuildStartedAt: new(int64), LastBuildFinishedAt: new(int64), LastCheckedIn: new(int64), diff --git a/go.mod b/go.mod index d0cf1c469..53e798357 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/vault/api v1.10.0 github.com/joho/godotenv v1.5.1 + github.com/lib/pq v1.10.9 + github.com/microcosm-cc/bluemonday v1.0.25 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.17.0 github.com/redis/go-redis/v9 v9.2.1 @@ -91,12 +93,10 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/mock/server/worker.go b/mock/server/worker.go index b8e1d724f..c930603db 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/library" ) @@ -27,6 +28,66 @@ const ( ], "active": true, "last_checked_in": 1602612590 + "running_builds": [ + { + "id": 2, + "repo_id": 1, + "number": 2, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474204, + "created": 1563474204, + "started": 1563474204, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "Second commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "ed95dcc0687c", + "runtime": "", + "distribution": "" + }, + { + "id": 1, + "repo_id": 1, + "number": 1, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474077, + "created": 1563474076, + "started": 1563474077, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "First commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "82823eb770b0", + "runtime": "", + "distribution": "" + } + ] }` // WorkersResp represents a JSON return for one to many workers. @@ -87,7 +148,7 @@ const ( func getWorkers(c *gin.Context) { data := []byte(WorkersResp) - var body []library.Worker + var body []api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -107,7 +168,7 @@ func getWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -139,7 +200,7 @@ func updateWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) diff --git a/router/middleware/logger_test.go b/router/middleware/logger_test.go index 4a699520a..6419d601f 100644 --- a/router/middleware/logger_test.go +++ b/router/middleware/logger_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/service" @@ -58,7 +59,7 @@ func TestMiddleware_Logger(t *testing.T) { u.SetName("foo") u.SetToken("bar") - w := new(library.Worker) + w := new(api.Worker) w.SetID(1) w.SetHostname("worker_0") w.SetAddress("localhost") diff --git a/router/middleware/worker/context.go b/router/middleware/worker/context.go index a7780a3e8..2e9772a9e 100644 --- a/router/middleware/worker/context.go +++ b/router/middleware/worker/context.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) const key = "worker" @@ -16,13 +16,13 @@ type Setter interface { } // FromContext returns the Worker associated with this context. -func FromContext(c context.Context) *library.Worker { +func FromContext(c context.Context) *api.Worker { value := c.Value(key) if value == nil { return nil } - w, ok := value.(*library.Worker) + w, ok := value.(*api.Worker) if !ok { return nil } @@ -32,6 +32,6 @@ func FromContext(c context.Context) *library.Worker { // ToContext adds the Worker to this context if it supports // the Setter interface. -func ToContext(c Setter, w *library.Worker) { +func ToContext(c Setter, w *api.Worker) { c.Set(key, w) } diff --git a/router/middleware/worker/context_test.go b/router/middleware/worker/context_test.go index 76efd687b..6fd731e90 100644 --- a/router/middleware/worker/context_test.go +++ b/router/middleware/worker/context_test.go @@ -5,7 +5,7 @@ package worker import ( "testing" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" "github.com/gin-gonic/gin" ) @@ -13,7 +13,7 @@ import ( func TestWorker_FromContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) @@ -72,7 +72,7 @@ func TestWorker_FromContext_Empty(t *testing.T) { func TestWorker_ToContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/worker/worker.go b/router/middleware/worker/worker.go index 11586af9c..5a78d14a4 100644 --- a/router/middleware/worker/worker.go +++ b/router/middleware/worker/worker.go @@ -7,14 +7,14 @@ import ( "net/http" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // Retrieve gets the worker in the given context. -func Retrieve(c *gin.Context) *library.Worker { +func Retrieve(c *gin.Context) *api.Worker { return FromContext(c) } diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index 4de2db921..a5b46b8ff 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -10,13 +10,14 @@ import ( "testing" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) func TestWorker_Retrieve(t *testing.T) { // setup types - want := new(library.Worker) + want := new(api.Worker) want.SetID(1) // setup context @@ -34,7 +35,10 @@ func TestWorker_Retrieve(t *testing.T) { func TestWorker_Establish(t *testing.T) { // setup types - want := new(library.Worker) + b := new(library.Build) + b.SetID(1) + + want := new(api.Worker) want.SetID(1) want.SetHostname("worker_0") want.SetAddress("localhost") @@ -42,13 +46,13 @@ func TestWorker_Establish(t *testing.T) { want.SetActive(true) want.SetStatus("available") want.SetLastStatusUpdateAt(12345) - want.SetRunningBuildIDs([]string{}) + want.SetRunningBuilds([]*library.Build{b}) want.SetLastBuildStartedAt(12345) want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) - got := new(library.Worker) + got := new(api.Worker) // setup database db, err := database.NewTest()