diff --git a/.github/workflows/validate-pr-title.yml b/.github/workflows/validate-pr-title.yml index 462f9be92..ec3674f79 100644 --- a/.github/workflows/validate-pr-title.yml +++ b/.github/workflows/validate-pr-title.yml @@ -14,4 +14,4 @@ jobs: steps: - name: validate title run: | - echo "${{ github.event.pull_request.title }}" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|):\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) + echo "${{ github.event.pull_request.title }}" | grep -Eq '^(feat|fix|chore|refactor|enhance|test|docs)(\(.*\)|)!?:\s.+$' && (echo "Pass"; exit 0) || (echo "Incorrect Format. Please see https://go-vela.github.io/docs/community/contributing_guidelines/#development-workflow"; exit 1) diff --git a/api/types/worker.go b/api/types/worker.go new file mode 100644 index 000000000..4c06a0e69 --- /dev/null +++ b/api/types/worker.go @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + + "github.com/go-vela/types/library" +) + +// Worker is the API representation of a worker. +// +// swagger:model Worker +type Worker struct { + ID *int64 `json:"id,omitempty"` + Hostname *string `json:"hostname,omitempty"` + Address *string `json:"address,omitempty"` + Routes *[]string `json:"routes,omitempty"` + Active *bool `json:"active,omitempty"` + Status *string `json:"status,omitempty"` + LastStatusUpdateAt *int64 `json:"last_status_update_at,omitempty"` + RunningBuilds *[]*library.Build `json:"running_builds,omitempty"` + LastBuildStartedAt *int64 `json:"last_build_started_at,omitempty"` + LastBuildFinishedAt *int64 `json:"last_build_finished_at,omitempty"` + LastCheckedIn *int64 `json:"last_checked_in,omitempty"` + BuildLimit *int64 `json:"build_limit,omitempty"` +} + +// GetID returns the ID field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetID() int64 { + // return zero value if Worker type or ID field is nil + if w == nil || w.ID == nil { + return 0 + } + + return *w.ID +} + +// GetHostname returns the Hostname field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetHostname() string { + // return zero value if Worker type or Hostname field is nil + if w == nil || w.Hostname == nil { + return "" + } + + return *w.Hostname +} + +// GetAddress returns the Address field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetAddress() string { + // return zero value if Worker type or Address field is nil + if w == nil || w.Address == nil { + return "" + } + + return *w.Address +} + +// GetRoutes returns the Routes field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRoutes() []string { + // return zero value if Worker type or Routes field is nil + if w == nil || w.Routes == nil { + return []string{} + } + + return *w.Routes +} + +// GetActive returns the Active field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetActive() bool { + // return zero value if Worker type or Active field is nil + if w == nil || w.Active == nil { + return false + } + + return *w.Active +} + +// GetStatus returns the Status field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetStatus() string { + // return zero value if Worker type or Status field is nil + if w == nil || w.Status == nil { + return "" + } + + return *w.Status +} + +// GetLastStatusUpdateAt returns the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastStatusUpdateAt() int64 { + // return zero value if Worker type or LastStatusUpdateAt field is nil + if w == nil || w.LastStatusUpdateAt == nil { + return 0 + } + + return *w.LastStatusUpdateAt +} + +// GetRunningBuilds returns the RunningBuilds field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetRunningBuilds() []*library.Build { + // return zero value if Worker type or RunningBuilds field is nil + if w == nil || w.RunningBuilds == nil { + return []*library.Build{} + } + + return *w.RunningBuilds +} + +// GetLastBuildStartedAt returns the LastBuildStartedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildStartedAt() int64 { + // return zero value if Worker type or LastBuildStartedAt field is nil + if w == nil || w.LastBuildStartedAt == nil { + return 0 + } + + return *w.LastBuildStartedAt +} + +// GetLastBuildFinishedAt returns the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastBuildFinishedAt() int64 { + // return zero value if Worker type or LastBuildFinishedAt field is nil + if w == nil || w.LastBuildFinishedAt == nil { + return 0 + } + + return *w.LastBuildFinishedAt +} + +// GetLastCheckedIn returns the LastCheckedIn field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetLastCheckedIn() int64 { + // return zero value if Worker type or LastCheckedIn field is nil + if w == nil || w.LastCheckedIn == nil { + return 0 + } + + return *w.LastCheckedIn +} + +// GetBuildLimit returns the BuildLimit field. +// +// When the provided Worker type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (w *Worker) GetBuildLimit() int64 { + // return zero value if Worker type or BuildLimit field is nil + if w == nil || w.BuildLimit == nil { + return 0 + } + + return *w.BuildLimit +} + +// SetID sets the ID field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetID(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.ID = &v +} + +// SetHostname sets the Hostname field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetHostname(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Hostname = &v +} + +// SetAddress sets the Address field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetAddress(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Address = &v +} + +// SetRoutes sets the Routes field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRoutes(v []string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Routes = &v +} + +// SetActive sets the Active field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetActive(v bool) { + // return if Worker type is nil + if w == nil { + return + } + + w.Active = &v +} + +// SetStatus sets the Status field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetStatus(v string) { + // return if Worker type is nil + if w == nil { + return + } + + w.Status = &v +} + +// SetLastStatusUpdateAt sets the LastStatusUpdateAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastStatusUpdateAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastStatusUpdateAt = &v +} + +// SetRunningBuilds sets the RunningBuilds field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetRunningBuilds(builds []*library.Build) { + // return if Worker type is nil + if w == nil { + return + } + + w.RunningBuilds = &builds +} + +// SetLastBuildStartedAt sets the LastBuildStartedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildStartedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildStartedAt = &v +} + +// SetLastBuildFinishedAt sets the LastBuildFinishedAt field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastBuildFinishedAt(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastBuildFinishedAt = &v +} + +// SetLastCheckedIn sets the LastCheckedIn field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetLastCheckedIn(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.LastCheckedIn = &v +} + +// SetBuildLimit sets the LastBuildLimit field. +// +// When the provided Worker type is nil, it +// will set nothing and immediately return. +func (w *Worker) SetBuildLimit(v int64) { + // return if Worker type is nil + if w == nil { + return + } + + w.BuildLimit = &v +} + +// String implements the Stringer interface for the Worker type. +func (w *Worker) String() string { + return fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) +} diff --git a/api/types/worker_test.go b/api/types/worker_test.go new file mode 100644 index 000000000..19d2defb5 --- /dev/null +++ b/api/types/worker_test.go @@ -0,0 +1,224 @@ +// SPDX-License-Identifier: Apache-2.0 + +package types + +import ( + "fmt" + "reflect" + "testing" + "time" + + "github.com/go-vela/types/library" +) + +func TestTypes_Worker_Getters(t *testing.T) { + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: new(Worker), + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + if test.worker.GetID() != test.want.GetID() { + t.Errorf("GetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("GetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("Getaddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("GetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("GetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("GetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("GetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if !reflect.DeepEqual(test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) { + t.Errorf("GetRunningBuildIDs is %v, want %v", test.worker.GetRunningBuilds(), test.want.GetRunningBuilds()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("GetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("GetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("GetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("GetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_Setters(t *testing.T) { + // setup types + var w *Worker + + // setup tests + tests := []struct { + worker *Worker + want *Worker + }{ + { + worker: testWorker(), + want: testWorker(), + }, + { + worker: w, + want: new(Worker), + }, + } + + // run tests + for _, test := range tests { + test.worker.SetID(test.want.GetID()) + test.worker.SetHostname(test.want.GetHostname()) + test.worker.SetAddress(test.want.GetAddress()) + test.worker.SetRoutes(test.want.GetRoutes()) + test.worker.SetActive(test.want.GetActive()) + test.worker.SetStatus(test.want.GetStatus()) + test.worker.SetLastStatusUpdateAt(test.want.GetLastStatusUpdateAt()) + test.worker.SetRunningBuilds(test.want.GetRunningBuilds()) + test.worker.SetLastBuildStartedAt(test.want.GetLastBuildStartedAt()) + test.worker.SetLastBuildFinishedAt(test.want.GetLastBuildFinishedAt()) + test.worker.SetLastCheckedIn(test.want.GetLastCheckedIn()) + test.worker.SetBuildLimit(test.want.GetBuildLimit()) + + if test.worker.GetID() != test.want.GetID() { + t.Errorf("SetID is %v, want %v", test.worker.GetID(), test.want.GetID()) + } + + if test.worker.GetHostname() != test.want.GetHostname() { + t.Errorf("SetHostname is %v, want %v", test.worker.GetHostname(), test.want.GetHostname()) + } + + if test.worker.GetAddress() != test.want.GetAddress() { + t.Errorf("SetAddress is %v, want %v", test.worker.GetAddress(), test.want.GetAddress()) + } + + if !reflect.DeepEqual(test.worker.GetRoutes(), test.want.GetRoutes()) { + t.Errorf("SetRoutes is %v, want %v", test.worker.GetRoutes(), test.want.GetRoutes()) + } + + if test.worker.GetActive() != test.want.GetActive() { + t.Errorf("SetActive is %v, want %v", test.worker.GetActive(), test.want.GetActive()) + } + + if test.worker.GetStatus() != test.want.GetStatus() { + t.Errorf("SetStatus is %v, want %v", test.worker.GetStatus(), test.want.GetStatus()) + } + + if test.worker.GetLastStatusUpdateAt() != test.want.GetLastStatusUpdateAt() { + t.Errorf("SetLastStatusUpdateAt is %v, want %v", test.worker.GetLastStatusUpdateAt(), test.want.GetLastStatusUpdateAt()) + } + + if test.worker.GetLastBuildStartedAt() != test.want.GetLastBuildStartedAt() { + t.Errorf("SetLastBuildStartedAt is %v, want %v", test.worker.GetLastBuildStartedAt(), test.want.GetLastBuildStartedAt()) + } + + if test.worker.GetLastBuildFinishedAt() != test.want.GetLastBuildFinishedAt() { + t.Errorf("SetLastBuildFinishedAt is %v, want %v", test.worker.GetLastBuildFinishedAt(), test.want.GetLastBuildFinishedAt()) + } + + if test.worker.GetLastCheckedIn() != test.want.GetLastCheckedIn() { + t.Errorf("SetLastCheckedIn is %v, want %v", test.worker.GetLastCheckedIn(), test.want.GetLastCheckedIn()) + } + + if test.worker.GetBuildLimit() != test.want.GetBuildLimit() { + t.Errorf("SetBuildLimit is %v, want %v", test.worker.GetBuildLimit(), test.want.GetBuildLimit()) + } + } +} + +func TestTypes_Worker_String(t *testing.T) { + // setup types + w := testWorker() + + want := fmt.Sprintf(`{ + ID: %d, + Hostname: %s, + Address: %s, + Routes: %s, + Active: %t, + Status: %s, + LastStatusUpdateAt: %v, + LastBuildStartedAt: %v, + LastBuildFinishedAt: %v, + LastCheckedIn: %v, + BuildLimit: %v, + RunningBuilds: %v, +}`, + w.GetID(), + w.GetHostname(), + w.GetAddress(), + w.GetRoutes(), + w.GetActive(), + w.GetStatus(), + w.GetLastStatusUpdateAt(), + w.GetLastBuildStartedAt(), + w.GetLastBuildFinishedAt(), + w.GetLastCheckedIn(), + w.GetBuildLimit(), + w.GetRunningBuilds(), + ) + + // run test + got := w.String() + + if !reflect.DeepEqual(got, want) { + t.Errorf("String is %v, want %v", got, want) + } +} + +// testWorker is a test helper function to create a Worker +// type with all fields set to a fake value. +func testWorker() *Worker { + b := new(library.Build) + b.SetID(1) + + w := new(Worker) + + w.SetID(1) + w.SetHostname("worker_0") + w.SetAddress("http://localhost:8080") + w.SetRoutes([]string{"vela"}) + w.SetActive(true) + w.SetStatus("available") + w.SetLastStatusUpdateAt(time.Time{}.UTC().Unix()) + w.SetRunningBuilds([]*library.Build{b}) + w.SetLastBuildStartedAt(time.Time{}.UTC().Unix()) + w.SetLastBuildFinishedAt(time.Time{}.UTC().Unix()) + w.SetLastCheckedIn(time.Time{}.UTC().Unix()) + w.SetBuildLimit(2) + + return w +} diff --git a/api/worker/create.go b/api/worker/create.go index 07edf3acc..d91898885 100644 --- a/api/worker/create.go +++ b/api/worker/create.go @@ -9,6 +9,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/router/middleware/claims" @@ -58,7 +59,7 @@ func CreateWorker(c *gin.Context) { ctx := c.Request.Context() // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { diff --git a/api/worker/get.go b/api/worker/get.go index a8a4d6931..b7bebd149 100644 --- a/api/worker/get.go +++ b/api/worker/get.go @@ -11,6 +11,7 @@ import ( "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -55,14 +56,21 @@ func GetWorker(c *gin.Context) { "worker": w.GetHostname(), }).Infof("reading worker %s", w.GetHostname()) - w, err := database.FromContext(c).GetWorkerForHostname(ctx, w.GetHostname()) - if err != nil { - retErr := fmt.Errorf("unable to get workers: %w", err) + rBs := []*library.Build{} - util.HandleError(c, http.StatusNotFound, retErr) + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) - return + return + } + + rBs = append(rBs, build) } + w.SetRunningBuilds(rBs) + c.JSON(http.StatusOK, w) } diff --git a/api/worker/list.go b/api/worker/list.go index f402144da..82673eccf 100644 --- a/api/worker/list.go +++ b/api/worker/list.go @@ -12,6 +12,7 @@ import ( "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/util" + "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -86,7 +87,7 @@ func ListWorkers(c *gin.Context) { return } - w, err := database.FromContext(c).ListWorkers(ctx, active, before, after) + workers, err := database.FromContext(c).ListWorkers(ctx, active, before, after) if err != nil { retErr := fmt.Errorf("unable to get workers: %w", err) @@ -95,5 +96,23 @@ func ListWorkers(c *gin.Context) { return } - c.JSON(http.StatusOK, w) + for _, w := range workers { + rBs := []*library.Build{} + + for _, b := range w.GetRunningBuilds() { + build, err := database.FromContext(c).GetBuild(ctx, b.GetID()) + if err != nil { + retErr := fmt.Errorf("unable to read build %d: %w", b.GetID(), err) + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + rBs = append(rBs, build) + } + + w.SetRunningBuilds(rBs) + } + + c.JSON(http.StatusOK, workers) } diff --git a/api/worker/update.go b/api/worker/update.go index d95ddf7af..e5b842a0d 100644 --- a/api/worker/update.go +++ b/api/worker/update.go @@ -7,11 +7,11 @@ import ( "net/http" "github.com/gin-gonic/gin" + "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/user" "github.com/go-vela/server/router/middleware/worker" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) @@ -71,7 +71,7 @@ func UpdateWorker(c *gin.Context) { }).Infof("updating worker %s", w.GetHostname()) // capture body from API request - input := new(library.Worker) + input := new(types.Worker) err := c.Bind(input) if err != nil { @@ -97,9 +97,9 @@ func UpdateWorker(c *gin.Context) { w.SetActive(input.GetActive()) } - if input.RunningBuildIDs != nil { + if input.RunningBuilds != nil { // update runningBuildIDs if set - w.SetRunningBuildIDs(input.GetRunningBuildIDs()) + w.SetRunningBuilds(input.GetRunningBuilds()) } if len(input.GetStatus()) > 0 { diff --git a/database/integration_test.go b/database/integration_test.go index 585cc4134..29ac6774d 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database/build" "github.com/go-vela/server/database/deployment" "github.com/go-vela/server/database/executable" @@ -43,7 +44,7 @@ type Resources struct { Services []*library.Service Steps []*library.Step Users []*library.User - Workers []*library.Worker + Workers []*api.Worker } func TestDatabase_Integration(t *testing.T) { @@ -2446,7 +2447,13 @@ func newResources() *Resources { userTwo.SetActive(true) userTwo.SetAdmin(false) - workerOne := new(library.Worker) + _bPartialOne := new(library.Build) + _bPartialOne.SetID(1) + + _bPartialTwo := new(library.Build) + _bPartialTwo.SetID(2) + + workerOne := new(api.Worker) workerOne.SetID(1) workerOne.SetHostname("worker-1.example.com") workerOne.SetAddress("https://worker-1.example.com") @@ -2454,13 +2461,13 @@ func newResources() *Resources { workerOne.SetActive(true) workerOne.SetStatus("available") workerOne.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerOne.SetRunningBuildIDs([]string{"12345"}) + workerOne.SetRunningBuilds([]*library.Build{_bPartialOne}) workerOne.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerOne.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerOne.SetLastCheckedIn(time.Now().UTC().Unix() - 60) workerOne.SetBuildLimit(1) - workerTwo := new(library.Worker) + workerTwo := new(api.Worker) workerTwo.SetID(2) workerTwo.SetHostname("worker-2.example.com") workerTwo.SetAddress("https://worker-2.example.com") @@ -2468,7 +2475,7 @@ func newResources() *Resources { workerTwo.SetActive(true) workerTwo.SetStatus("available") workerTwo.SetLastStatusUpdateAt(time.Now().UTC().Unix()) - workerTwo.SetRunningBuildIDs([]string{"12345"}) + workerTwo.SetRunningBuilds([]*library.Build{_bPartialTwo}) workerTwo.SetLastBuildStartedAt(time.Now().UTC().Unix()) workerTwo.SetLastBuildFinishedAt(time.Now().UTC().Unix()) workerTwo.SetLastCheckedIn(time.Now().UTC().Unix() - 60) @@ -2487,7 +2494,7 @@ func newResources() *Resources { Services: []*library.Service{serviceOne, serviceTwo}, Steps: []*library.Step{stepOne, stepTwo}, Users: []*library.User{userOne, userTwo}, - Workers: []*library.Worker{workerOne, workerTwo}, + Workers: []*api.Worker{workerOne, workerTwo}, } } diff --git a/database/worker/create.go b/database/worker/create.go index 12b386319..131bb393f 100644 --- a/database/worker/create.go +++ b/database/worker/create.go @@ -5,14 +5,13 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // CreateWorker creates a new worker in the database. -func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) CreateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("creating worker %s in the database", w.GetHostname()) @@ -20,7 +19,7 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := FromAPI(w) // validate the necessary fields are populated // @@ -33,5 +32,5 @@ func (e *engine) CreateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Create(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/create_test.go b/database/worker/create_test.go index dad81473a..6809bb33e 100644 --- a/database/worker/create_test.go +++ b/database/worker/create_test.go @@ -28,7 +28,7 @@ func TestWorker_Engine_CreateWorker(t *testing.T) { _mock.ExpectQuery(`INSERT INTO "workers" ("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id") VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/worker/delete.go b/database/worker/delete.go index ba846db91..ee391567e 100644 --- a/database/worker/delete.go +++ b/database/worker/delete.go @@ -5,14 +5,13 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // DeleteWorker deletes an existing worker from the database. -func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { +func (e *engine) DeleteWorker(ctx context.Context, w *api.Worker) error { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("deleting worker %s from the database", w.GetHostname()) @@ -20,7 +19,7 @@ func (e *engine) DeleteWorker(ctx context.Context, w *library.Worker) error { // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := FromAPI(w) // send query to the database return e.client. diff --git a/database/worker/get.go b/database/worker/get.go index 12201d4d5..7531fd05a 100644 --- a/database/worker/get.go +++ b/database/worker/get.go @@ -5,17 +5,16 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // GetWorker gets a worker by ID from the database. -func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, error) { +func (e *engine) GetWorker(ctx context.Context, id int64) (*api.Worker, error) { e.logger.Tracef("getting worker %d from the database", id) // variable to store query results - w := new(database.Worker) + w := new(Worker) // send query to the database and store result in variable err := e.client. @@ -30,5 +29,5 @@ func (e *engine) GetWorker(ctx context.Context, id int64) (*library.Worker, erro // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname.go b/database/worker/get_hostname.go index 3370620a0..f596850af 100644 --- a/database/worker/get_hostname.go +++ b/database/worker/get_hostname.go @@ -5,20 +5,19 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // GetWorkerForHostname gets a worker by hostname from the database. -func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*library.Worker, error) { +func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": hostname, }).Tracef("getting worker %s from the database", hostname) // variable to store query results - w := new(database.Worker) + w := new(Worker) // send query to the database and store result in variable err := e.client. @@ -33,5 +32,5 @@ func (e *engine) GetWorkerForHostname(ctx context.Context, hostname string) (*li // return the worker // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - return w.ToLibrary(), nil + return w.ToAPI(convertToBuilds(w.RunningBuildIDs)), nil } diff --git a/database/worker/get_hostname_test.go b/database/worker/get_hostname_test.go index cf264ab0c..7d5043340 100644 --- a/database/worker/get_hostname_test.go +++ b/database/worker/get_hostname_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorkerForName(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) // sqlmock cannot parse string array values _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/get_test.go b/database/worker/get_test.go index bfd9bf2a3..566f8232b 100644 --- a/database/worker/get_test.go +++ b/database/worker/get_test.go @@ -8,7 +8,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_Engine_GetWorker(t *testing.T) { @@ -18,6 +18,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { _worker.SetHostname("worker_0") _worker.SetAddress("localhost") _worker.SetActive(true) + _worker.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -43,7 +44,7 @@ func TestWorker_Engine_GetWorker(t *testing.T) { failure bool name string database *engine - want *library.Worker + want *api.Worker }{ { failure: false, diff --git a/database/worker/interface.go b/database/worker/interface.go index 589e8971b..8f8d024a1 100644 --- a/database/worker/interface.go +++ b/database/worker/interface.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) // WorkerInterface represents the Vela interface for worker @@ -29,15 +29,15 @@ type WorkerInterface interface { // CountWorkers defines a function that gets the count of all workers. CountWorkers(context.Context) (int64, error) // CreateWorker defines a function that creates a new worker. - CreateWorker(context.Context, *library.Worker) (*library.Worker, error) + CreateWorker(context.Context, *api.Worker) (*api.Worker, error) // DeleteWorker defines a function that deletes an existing worker. - DeleteWorker(context.Context, *library.Worker) error + DeleteWorker(context.Context, *api.Worker) error // GetWorker defines a function that gets a worker by ID. - GetWorker(context.Context, int64) (*library.Worker, error) + GetWorker(context.Context, int64) (*api.Worker, error) // GetWorkerForHostname defines a function that gets a worker by hostname. - GetWorkerForHostname(context.Context, string) (*library.Worker, error) + GetWorkerForHostname(context.Context, string) (*api.Worker, error) // ListWorkers defines a function that gets a list of all workers. - ListWorkers(context.Context, string, int64, int64) ([]*library.Worker, error) + ListWorkers(context.Context, string, int64, int64) ([]*api.Worker, error) // UpdateWorker defines a function that updates an existing worker. - UpdateWorker(context.Context, *library.Worker) (*library.Worker, error) + UpdateWorker(context.Context, *api.Worker) (*api.Worker, error) } diff --git a/database/worker/list.go b/database/worker/list.go index bb8169d61..daa9a7e06 100644 --- a/database/worker/list.go +++ b/database/worker/list.go @@ -7,18 +7,17 @@ import ( "fmt" "strconv" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" ) // ListWorkers gets a list of all workers from the database. -func (e *engine) ListWorkers(ctx context.Context, active string, before, after int64) ([]*library.Worker, error) { +func (e *engine) ListWorkers(ctx context.Context, active string, before, after int64) ([]*api.Worker, error) { e.logger.Trace("listing all workers from the database") // variables to store query results and return value - w := new([]database.Worker) - workers := []*library.Worker{} + results := new([]Worker) + workers := []*api.Worker{} // build query with checked in constraints query := e.client.Table(constants.TableWorker). @@ -37,20 +36,20 @@ func (e *engine) ListWorkers(ctx context.Context, active string, before, after i } // send query to the database and store result in variable - err := query.Find(&w).Error + err := query.Find(&results).Error if err != nil { return nil, err } // iterate through all query results - for _, worker := range *w { + for _, worker := range *results { // https://golang.org/doc/faq#closures_and_goroutines tmp := worker // convert query result to library type // // https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary - workers = append(workers, tmp.ToLibrary()) + workers = append(workers, tmp.ToAPI(convertToBuilds(tmp.RunningBuildIDs))) } return workers, nil diff --git a/database/worker/list_test.go b/database/worker/list_test.go index 4a5988228..4de8883a6 100644 --- a/database/worker/list_test.go +++ b/database/worker/list_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" "github.com/google/go-cmp/cmp" ) @@ -21,6 +21,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { _workerOne.SetHostname("worker_0") _workerOne.SetAddress("localhost") _workerOne.SetActive(true) + _workerOne.SetRunningBuilds(nil) _workerOne.SetLastCheckedIn(newer) _workerTwo := testWorker() @@ -29,6 +30,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { _workerTwo.SetAddress("localhost") _workerTwo.SetActive(true) _workerTwo.SetLastCheckedIn(older) + _workerTwo.SetRunningBuilds(nil) _workerThree := testWorker() _workerThree.SetID(3) @@ -36,6 +38,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { _workerThree.SetAddress("localhost") _workerThree.SetActive(false) _workerThree.SetLastCheckedIn(newer) + _workerThree.SetRunningBuilds(nil) _postgres, _mock := testPostgres(t) defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() @@ -75,7 +78,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { active string name string database *engine - want []*library.Worker + want []*api.Worker }{ { failure: false, @@ -83,7 +86,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { active: "all", name: "sqlite3 before filter", database: _sqlite, - want: []*library.Worker{_workerTwo}, + want: []*api.Worker{_workerTwo}, }, { failure: false, @@ -91,7 +94,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { active: "all", name: "postgres catch all", database: _postgres, - want: []*library.Worker{_workerOne, _workerTwo, _workerThree}, + want: []*api.Worker{_workerOne, _workerTwo, _workerThree}, }, { failure: false, @@ -99,7 +102,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { active: "all", name: "sqlite3 catch all", database: _sqlite, - want: []*library.Worker{_workerOne, _workerTwo, _workerThree}, + want: []*api.Worker{_workerOne, _workerTwo, _workerThree}, }, { failure: false, @@ -107,7 +110,7 @@ func TestWorker_Engine_ListWorkers(t *testing.T) { active: "true", name: "sqlite3 active filter", database: _sqlite, - want: []*library.Worker{_workerOne, _workerTwo}, + want: []*api.Worker{_workerOne, _workerTwo}, }, } diff --git a/database/worker/update.go b/database/worker/update.go index 3cae0d164..0349ffe85 100644 --- a/database/worker/update.go +++ b/database/worker/update.go @@ -5,14 +5,13 @@ package worker import ( "context" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // UpdateWorker updates an existing worker in the database. -func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library.Worker, error) { +func (e *engine) UpdateWorker(ctx context.Context, w *api.Worker) (*api.Worker, error) { e.logger.WithFields(logrus.Fields{ "worker": w.GetHostname(), }).Tracef("updating worker %s in the database", w.GetHostname()) @@ -20,7 +19,7 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // cast the library type to database type // // https://pkg.go.dev/github.com/go-vela/types/database#WorkerFromLibrary - worker := database.WorkerFromLibrary(w) + worker := FromAPI(w) // validate the necessary fields are populated // @@ -33,5 +32,5 @@ func (e *engine) UpdateWorker(ctx context.Context, w *library.Worker) (*library. // send query to the database result := e.client.Table(constants.TableWorker).Save(worker) - return worker.ToLibrary(), result.Error + return worker.ToAPI(w.GetRunningBuilds()), result.Error } diff --git a/database/worker/update_test.go b/database/worker/update_test.go index d077bc567..5a483660b 100644 --- a/database/worker/update_test.go +++ b/database/worker/update_test.go @@ -25,7 +25,7 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) { _mock.ExpectExec(`UPDATE "workers" SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11 WHERE "id" = $12`). - WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1). + WithArgs("worker_0", "localhost", nil, true, nil, nil, `{"1"}`, nil, nil, nil, nil, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/worker/worker.go b/database/worker/worker.go index 09b69176a..d12055ded 100644 --- a/database/worker/worker.go +++ b/database/worker/worker.go @@ -4,14 +4,35 @@ package worker import ( "context" + "database/sql" + "errors" "fmt" + "strconv" + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/util" "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" + "github.com/lib/pq" "github.com/sirupsen/logrus" "gorm.io/gorm" ) +var ( + // ErrEmptyWorkerHost defines the error type when a + // Worker type has an empty Host field provided. + ErrEmptyWorkerHost = errors.New("empty worker hostname provided") + + // ErrEmptyWorkerAddress defines the error type when a + // Worker type has an empty Address field provided. + ErrEmptyWorkerAddress = errors.New("empty worker address provided") + + // ErrExceededRunningBuildIDsLimit defines the error type when a + // Worker type has RunningBuildIDs field provided that exceeds the database limit. + ErrExceededRunningBuildIDsLimit = errors.New("exceeded running build ids limit") +) + type ( // config represents the settings required to create the engine that implements the WorkerInterface interface. config struct { @@ -36,6 +57,22 @@ type ( // https://pkg.go.dev/github.com/sirupsen/logrus#Entry logger *logrus.Entry } + + // Worker is the database representation of a worker. + Worker struct { + ID sql.NullInt64 `sql:"id"` + Hostname sql.NullString `sql:"hostname"` + Address sql.NullString `sql:"address"` + Routes pq.StringArray `sql:"routes" gorm:"type:varchar(1000)"` + Active sql.NullBool `sql:"active"` + Status sql.NullString `sql:"status"` + LastStatusUpdateAt sql.NullInt64 `sql:"last_status_update_at"` + RunningBuildIDs pq.StringArray `sql:"running_build_ids" gorm:"type:varchar(500)"` + LastBuildStartedAt sql.NullInt64 `sql:"last_build_started_at"` + LastBuildFinishedAt sql.NullInt64 `sql:"last_build_finished_at"` + LastCheckedIn sql.NullInt64 `sql:"last_checked_in"` + BuildLimit sql.NullInt64 `sql:"build_limit"` + } ) // New creates and returns a Vela service for integrating with workers in the database. @@ -79,3 +116,170 @@ func New(opts ...EngineOpt) (*engine, error) { return e, nil } + +// Nullify ensures the valid flag for +// the sql.Null types are properly set. +// +// When a field within the Build type is the zero +// value for the field, the valid flag is set to +// false causing it to be NULL in the database. +func (w *Worker) Nullify() *Worker { + if w == nil { + return nil + } + + // check if the ID field should be false + if w.ID.Int64 == 0 { + w.ID.Valid = false + } + + // check if the Hostname field should be false + if len(w.Hostname.String) == 0 { + w.Hostname.Valid = false + } + + // check if the Address field should be false + if len(w.Address.String) == 0 { + w.Address.Valid = false + } + + // check if the Status field should be false + if len(w.Status.String) == 0 { + w.Status.Valid = false + } + + // check if the LastStatusUpdateAt field should be false + if w.LastStatusUpdateAt.Int64 == 0 { + w.LastStatusUpdateAt.Valid = false + } + + // check if the LastBuildStartedAt field should be false + if w.LastBuildStartedAt.Int64 == 0 { + w.LastBuildStartedAt.Valid = false + } + + // check if the LastBuildFinishedAt field should be false + if w.LastBuildFinishedAt.Int64 == 0 { + w.LastBuildFinishedAt.Valid = false + } + + // check if the LastCheckedIn field should be false + if w.LastCheckedIn.Int64 == 0 { + w.LastCheckedIn.Valid = false + } + + if w.BuildLimit.Int64 == 0 { + w.BuildLimit.Valid = false + } + + return w +} + +// ToAPI converts the Worker type +// to an API Worker type. +func (w *Worker) ToAPI(builds []*library.Build) *api.Worker { + worker := new(api.Worker) + + worker.SetID(w.ID.Int64) + worker.SetHostname(w.Hostname.String) + worker.SetAddress(w.Address.String) + worker.SetRoutes(w.Routes) + worker.SetActive(w.Active.Bool) + worker.SetStatus(w.Status.String) + worker.SetLastStatusUpdateAt(w.LastStatusUpdateAt.Int64) + worker.SetRunningBuilds(builds) + worker.SetLastBuildStartedAt(w.LastBuildStartedAt.Int64) + worker.SetLastBuildFinishedAt(w.LastBuildFinishedAt.Int64) + worker.SetLastCheckedIn(w.LastCheckedIn.Int64) + worker.SetBuildLimit(w.BuildLimit.Int64) + + return worker +} + +// Validate verifies the necessary fields for +// the Worker type are populated correctly. +func (w *Worker) Validate() error { + // verify the Host field is populated + if len(w.Hostname.String) == 0 { + return ErrEmptyWorkerHost + } + + // verify the Address field is populated + if len(w.Address.String) == 0 { + return ErrEmptyWorkerAddress + } + + // calculate total size of RunningBuildIds + total := 0 + for _, f := range w.RunningBuildIDs { + total += len(f) + } + + // verify the RunningBuildIds field is within the database constraints + // len is to factor in number of comma separators included in the database field, + // removing 1 due to the last item not having an appended comma + if (total + len(w.RunningBuildIDs) - 1) > constants.RunningBuildIDsMaxSize { + return ErrExceededRunningBuildIDsLimit + } + + // ensure that all Worker string fields + // that can be returned as JSON are sanitized + // to avoid unsafe HTML content + w.Hostname = sql.NullString{String: util.Sanitize(w.Hostname.String), Valid: w.Hostname.Valid} + w.Address = sql.NullString{String: util.Sanitize(w.Address.String), Valid: w.Address.Valid} + + // ensure that all Routes are sanitized + // to avoid unsafe HTML content + for i, v := range w.Routes { + w.Routes[i] = util.Sanitize(v) + } + + return nil +} + +// FromAPI converts the API worker type +// to a database worker type. +func FromAPI(w *api.Worker) *Worker { + var rBs []string + + for _, b := range w.GetRunningBuilds() { + rBs = append(rBs, fmt.Sprint(b.GetID())) + } + + worker := &Worker{ + ID: sql.NullInt64{Int64: w.GetID(), Valid: true}, + Hostname: sql.NullString{String: w.GetHostname(), Valid: true}, + Address: sql.NullString{String: w.GetAddress(), Valid: true}, + Routes: pq.StringArray(w.GetRoutes()), + Active: sql.NullBool{Bool: w.GetActive(), Valid: true}, + Status: sql.NullString{String: w.GetStatus(), Valid: true}, + LastStatusUpdateAt: sql.NullInt64{Int64: w.GetLastStatusUpdateAt(), Valid: true}, + RunningBuildIDs: pq.StringArray(rBs), + LastBuildStartedAt: sql.NullInt64{Int64: w.GetLastBuildStartedAt(), Valid: true}, + LastBuildFinishedAt: sql.NullInt64{Int64: w.GetLastBuildFinishedAt(), Valid: true}, + LastCheckedIn: sql.NullInt64{Int64: w.GetLastCheckedIn(), Valid: true}, + BuildLimit: sql.NullInt64{Int64: w.GetBuildLimit(), Valid: true}, + } + + return worker.Nullify() +} + +// convertToBuilds is a helper function that generates build objects with ID fields given a list of IDs. +func convertToBuilds(ids []string) []*library.Build { + // create stripped build objects holding the IDs + var rBs []*library.Build + + for _, b := range ids { + id, err := strconv.ParseInt(b, 10, 64) + if err != nil { + return nil + } + + build := new(library.Build) + build.SetID(id) + + rBs = append(rBs, build) + } + + return rBs +} diff --git a/database/worker/worker_test.go b/database/worker/worker_test.go index a219d1ffc..8c6896eb7 100644 --- a/database/worker/worker_test.go +++ b/database/worker/worker_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types/library" "github.com/sirupsen/logrus" @@ -106,6 +107,48 @@ func TestWorker_New(t *testing.T) { } } +func TestWorker_convertToBuilds(t *testing.T) { + _buildOne := new(library.Build) + _buildOne.SetID(1) + + _buildTwo := new(library.Build) + _buildTwo.SetID(2) + + // setup tests + tests := []struct { + name string + ids []string + want []*library.Build + }{ + { + name: "one id", + ids: []string{"1"}, + want: []*library.Build{_buildOne}, + }, + { + name: "multiple ids", + ids: []string{"1", "2"}, + want: []*library.Build{_buildOne, _buildTwo}, + }, + { + name: "not int64", + ids: []string{"1", "foo"}, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := convertToBuilds(test.ids) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("convertToBuilds for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + // testPostgres is a helper function to create a Postgres engine for testing. func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { // create the new mock sql database @@ -166,8 +209,11 @@ func testSqlite(t *testing.T) *engine { // testWorker is a test helper function to create a library // Worker type with all fields set to their zero values. -func testWorker() *library.Worker { - return &library.Worker{ +func testWorker() *api.Worker { + b := new(library.Build) + b.SetID(1) + + return &api.Worker{ ID: new(int64), Hostname: new(string), Address: new(string), @@ -175,7 +221,7 @@ func testWorker() *library.Worker { Active: new(bool), Status: new(string), LastStatusUpdateAt: new(int64), - RunningBuildIDs: new([]string), + RunningBuilds: &[]*library.Build{b}, LastBuildStartedAt: new(int64), LastBuildFinishedAt: new(int64), LastCheckedIn: new(int64), diff --git a/go.mod b/go.mod index 745dcc4c5..7d043ce7f 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/vault/api v1.12.1 github.com/joho/godotenv v1.5.1 + github.com/lib/pq v1.10.9 + github.com/microcosm-cc/bluemonday v1.0.26 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.0 github.com/redis/go-redis/v9 v9.5.1 @@ -89,10 +91,8 @@ require ( github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect - github.com/microcosm-cc/bluemonday v1.0.26 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/mock/server/worker.go b/mock/server/worker.go index 6abf38451..edc758460 100644 --- a/mock/server/worker.go +++ b/mock/server/worker.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/types" "github.com/go-vela/types/library" ) @@ -27,12 +28,71 @@ const ( ], "active": true, "last_checked_in": 1602612590, - "status": "idle", - "last_status_update_at": 160000000, - "running_build_ids": [], - "last_build_started_at": 1, - "last_build_finished_at": 2, - "build_limit": 1 + "status": "busy", + "last_status_update_at": 1602612590, + "last_build_started_at": 1602612590, + "last_build_finished_at": 1602612590, + "build_limit": 2, + "running_builds": [ + { + "id": 2, + "repo_id": 1, + "number": 2, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474204, + "created": 1563474204, + "started": 1563474204, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "Second commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "ed95dcc0687c", + "runtime": "", + "distribution": "" + }, + { + "id": 1, + "repo_id": 1, + "number": 1, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474077, + "created": 1563474076, + "started": 1563474077, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "First commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "82823eb770b0", + "runtime": "", + "distribution": "" + } + ] }` // WorkersResp represents a JSON return for one to many workers. @@ -47,7 +107,43 @@ const ( "large:docker" ], "active": true, - "last_checked_in": 1602612590 + "last_checked_in": 1602612590, + "status": "available", + "last_status_update_at": 1602612590, + "last_build_started_at": 1602612590, + "last_build_finished_at": 1602612590, + "build_limit": 2, + "running_builds": [ + { + "id": 2, + "repo_id": 1, + "number": 2, + "parent": 1, + "event": "push", + "status": "running", + "error": "", + "enqueued": 1563474204, + "created": 1563474204, + "started": 1563474204, + "finished": 0, + "deploy": "", + "clone": "https://github.com/github/octocat.git", + "source": "https://github.com/github/octocat/commit/48afb5bdc41ad69bf22588491333f7cf71135163", + "title": "push received from https://github.com/github/octocat", + "message": "Second commit...", + "commit": "48afb5bdc41ad69bf22588491333f7cf71135163", + "sender": "OctoKitty", + "author": "OctoKitty", + "email": "octokitty@github.com", + "link": "https://vela.example.company.com/github/octocat/1", + "branch": "main", + "ref": "refs/heads/main", + "base_ref": "", + "host": "ed95dcc0687c", + "runtime": "", + "distribution": "" + } + ] }, { "id": 2, @@ -59,7 +155,13 @@ const ( "large:docker" ], "active": true, - "last_checked_in": 1602612590 + "last_checked_in": 1602612590, + "status": "idle", + "last_status_update_at": 1602612590, + "last_build_started_at": 1602612590, + "last_build_finished_at": 1602612590, + "build_limit": 2, + "running_builds": [] } ]` @@ -93,7 +195,7 @@ const ( func getWorkers(c *gin.Context) { data := []byte(WorkersResp) - var body []library.Worker + var body []api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -113,7 +215,7 @@ func getWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) @@ -145,7 +247,7 @@ func updateWorker(c *gin.Context) { data := []byte(WorkerResp) - var body library.Worker + var body api.Worker _ = json.Unmarshal(data, &body) c.JSON(http.StatusOK, body) diff --git a/mock/server/worker_test.go b/mock/server/worker_test.go index defc3bd86..a9a46b810 100644 --- a/mock/server/worker_test.go +++ b/mock/server/worker_test.go @@ -7,11 +7,11 @@ import ( "reflect" "testing" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) func TestWorker_ActiveWorkerResp(t *testing.T) { - testWorker := library.Worker{} + testWorker := api.Worker{} err := json.Unmarshal([]byte(WorkerResp), &testWorker) if err != nil { @@ -26,3 +26,22 @@ func TestWorker_ActiveWorkerResp(t *testing.T) { } } } + +func TestWorker_ListActiveWorkerResp(t *testing.T) { + testWorkers := []api.Worker{} + + err := json.Unmarshal([]byte(WorkersResp), &testWorkers) + if err != nil { + t.Errorf("error unmarshaling worker: %v", err) + } + + for index, worker := range testWorkers { + tWorker := reflect.TypeOf(worker) + + for i := 0; i < tWorker.NumField(); i++ { + if reflect.ValueOf(worker).Field(i).IsNil() { + t.Errorf("WorkersResp index %d missing field %s", index, tWorker.Field(i).Name) + } + } + } +} diff --git a/router/middleware/logger_test.go b/router/middleware/logger_test.go index 9264ade1a..bfc084437 100644 --- a/router/middleware/logger_test.go +++ b/router/middleware/logger_test.go @@ -14,6 +14,7 @@ import ( "time" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/router/middleware/build" "github.com/go-vela/server/router/middleware/repo" "github.com/go-vela/server/router/middleware/service" @@ -58,7 +59,7 @@ func TestMiddleware_Logger(t *testing.T) { u.SetName("foo") u.SetToken("bar") - w := new(library.Worker) + w := new(api.Worker) w.SetID(1) w.SetHostname("worker_0") w.SetAddress("localhost") diff --git a/router/middleware/worker/context.go b/router/middleware/worker/context.go index a7780a3e8..2e9772a9e 100644 --- a/router/middleware/worker/context.go +++ b/router/middleware/worker/context.go @@ -5,7 +5,7 @@ package worker import ( "context" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" ) const key = "worker" @@ -16,13 +16,13 @@ type Setter interface { } // FromContext returns the Worker associated with this context. -func FromContext(c context.Context) *library.Worker { +func FromContext(c context.Context) *api.Worker { value := c.Value(key) if value == nil { return nil } - w, ok := value.(*library.Worker) + w, ok := value.(*api.Worker) if !ok { return nil } @@ -32,6 +32,6 @@ func FromContext(c context.Context) *library.Worker { // ToContext adds the Worker to this context if it supports // the Setter interface. -func ToContext(c Setter, w *library.Worker) { +func ToContext(c Setter, w *api.Worker) { c.Set(key, w) } diff --git a/router/middleware/worker/context_test.go b/router/middleware/worker/context_test.go index 76efd687b..6fd731e90 100644 --- a/router/middleware/worker/context_test.go +++ b/router/middleware/worker/context_test.go @@ -5,7 +5,7 @@ package worker import ( "testing" - "github.com/go-vela/types/library" + api "github.com/go-vela/server/api/types" "github.com/gin-gonic/gin" ) @@ -13,7 +13,7 @@ import ( func TestWorker_FromContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) @@ -72,7 +72,7 @@ func TestWorker_FromContext_Empty(t *testing.T) { func TestWorker_ToContext(t *testing.T) { // setup types num := int64(1) - want := &library.Worker{ID: &num} + want := &api.Worker{ID: &num} // setup context gin.SetMode(gin.TestMode) diff --git a/router/middleware/worker/worker.go b/router/middleware/worker/worker.go index 11586af9c..5a78d14a4 100644 --- a/router/middleware/worker/worker.go +++ b/router/middleware/worker/worker.go @@ -7,14 +7,14 @@ import ( "net/http" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/server/util" - "github.com/go-vela/types/library" "github.com/sirupsen/logrus" ) // Retrieve gets the worker in the given context. -func Retrieve(c *gin.Context) *library.Worker { +func Retrieve(c *gin.Context) *api.Worker { return FromContext(c) } diff --git a/router/middleware/worker/worker_test.go b/router/middleware/worker/worker_test.go index 2d3578cd0..277ed8c09 100644 --- a/router/middleware/worker/worker_test.go +++ b/router/middleware/worker/worker_test.go @@ -10,13 +10,14 @@ import ( "testing" "github.com/gin-gonic/gin" + api "github.com/go-vela/server/api/types" "github.com/go-vela/server/database" "github.com/go-vela/types/library" ) func TestWorker_Retrieve(t *testing.T) { // setup types - want := new(library.Worker) + want := new(api.Worker) want.SetID(1) // setup context @@ -34,7 +35,10 @@ func TestWorker_Retrieve(t *testing.T) { func TestWorker_Establish(t *testing.T) { // setup types - want := new(library.Worker) + b := new(library.Build) + b.SetID(1) + + want := new(api.Worker) want.SetID(1) want.SetHostname("worker_0") want.SetAddress("localhost") @@ -42,13 +46,13 @@ func TestWorker_Establish(t *testing.T) { want.SetActive(true) want.SetStatus("available") want.SetLastStatusUpdateAt(12345) - want.SetRunningBuildIDs([]string{}) + want.SetRunningBuilds([]*library.Build{b}) want.SetLastBuildStartedAt(12345) want.SetLastBuildFinishedAt(12345) want.SetLastCheckedIn(12345) want.SetBuildLimit(0) - got := new(library.Worker) + got := new(api.Worker) // setup database db, err := database.NewTest() diff --git a/util/util.go b/util/util.go index 840dbc04f..5d9ce644f 100644 --- a/util/util.go +++ b/util/util.go @@ -4,9 +4,11 @@ package util import ( "html" + "net/url" "strings" "github.com/go-vela/types/library" + "github.com/microcosm-cc/bluemonday" "github.com/gin-gonic/gin" "github.com/go-vela/types" @@ -128,3 +130,34 @@ func CheckAllowlist(r *library.Repo, allowlist []string) bool { return false } + +// Sanitize is a helper function to verify the provided input +// field does not contain HTML content. If the input field +// does contain HTML, then the function will sanitize and +// potentially remove the HTML if deemed malicious. +func Sanitize(field string) string { + // create new HTML input microcosm-cc/bluemonday policy + p := bluemonday.StrictPolicy() + + // create a URL query unescaped string from the field + queryUnescaped, err := url.QueryUnescape(field) + if err != nil { + // overwrite URL query unescaped string with field + queryUnescaped = field + } + + // create an HTML escaped string from the field + htmlEscaped := html.EscapeString(queryUnescaped) + + // create a microcosm-cc/bluemonday escaped string from the field + bluemondayEscaped := p.Sanitize(queryUnescaped) + + // check if the field contains html + if !strings.EqualFold(htmlEscaped, bluemondayEscaped) { + // create new HTML input microcosm-cc/bluemonday policy + return bluemondayEscaped + } + + // return the unmodified field + return field +} diff --git a/util/util_test.go b/util/util_test.go new file mode 100644 index 000000000..bf4c458f1 --- /dev/null +++ b/util/util_test.go @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: Apache-2.0 + +package util + +import ( + "testing" +) + +func TestUtil_Sanitize(t *testing.T) { + // setup tests + tests := []struct { + name string + value string + want string + }{ + { + name: "percent", + value: `%`, + want: `%`, + }, + { + name: "quoted", + value: `"hello"`, + want: `"hello"`, + }, + { + name: "email", + value: `OctoKitty@github.com`, + want: `OctoKitty@github.com`, + }, + { + name: "url", + value: `https://github.com/go-vela`, + want: `https://github.com/go-vela`, + }, + { + name: "encoded", + value: `+ added foo %25 + updated bar %22 +`, + want: `+ added foo %25 + updated bar %22 +`, + }, + { + name: "html with headers", + value: `Merge pull request #1 from me/patch-1\n\n

hello

is now

hello

`, + want: `Merge pull request #1 from me/patch-1\n\nhello is now hello`, + }, + { + name: "html with email", + value: `Co-authored-by: OctoKitty `, + want: `Co-authored-by: OctoKitty `, + }, + { + name: "html with href", + value: `Google`, + want: `Google`, + }, + { + name: "local cross-site script", + value: ``, + want: ``, + }, + { + name: "remote cross-site script", + value: ``, + want: ``, + }, + { + name: "embedded cross-site script", + value: `%3cDIV%20STYLE%3d%22width%3a%20expression(alert('XSS'))%3b%22%3e`, + want: ``, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := Sanitize(test.value) + + if got != test.want { + t.Errorf("sanitize is %v, want %v", got, test.want) + } + }) + } +}