Skip to content

Commit

Permalink
feat(worker-visibility): extend worker table with 5 fields (#772)
Browse files Browse the repository at this point in the history
  • Loading branch information
KellyMerrick authored Jun 16, 2023
1 parent e17c971 commit 4e5d484
Show file tree
Hide file tree
Showing 13 changed files with 181 additions and 54 deletions.
84 changes: 79 additions & 5 deletions api/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/go-vela/server/database"
"github.com/go-vela/server/queue"
"github.com/go-vela/types/constants"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/prometheus/client_golang/prometheus/promhttp"
Expand Down Expand Up @@ -56,6 +57,18 @@ type MetricsQueryParameters struct {
ActiveWorkerCount bool `form:"active_worker_count"`
// InactiveWorkerCount represents total number of inactive workers
InactiveWorkerCount bool `form:"inactive_worker_count"`

// IdleWorkerCount represents total number of workers with a status of idle
// where worker RunningBuildIDs.length = 0
IdleWorkerCount bool `form:"idle_worker_count"`
// AvailableWorkerCount represents total number of workers with a status of available,
// where worker RunningBuildIDs.length > 0 and < worker BuildLimit
AvailableWorkerCount bool `form:"available_worker_count"`
// BusyWorkerCount represents total number of workers with a status of busy,
// where worker BuildLimit == worker RunningBuildIDs.length
BusyWorkerCount bool `form:"busy_worker_count"`
// ErrorWorkerCount represents total number of workers with a status of error
ErrorWorkerCount bool `form:"error_worker_count"`
}

// predefine Prometheus metrics else they will be regenerated
Expand Down Expand Up @@ -180,6 +193,26 @@ var (
// description: Indicates a request for inactive worker count
// type: boolean
// default: false
// - in: query
// name: idle_worker_count
// description: Indicates a request for idle worker count
// type: boolean
// default: false
// - in: query
// name: available_worker_count
// description: Indicates a request for available worker count
// type: boolean
// default: false
// - in: query
// name: busy_worker_count
// description: Indicates a request for busy worker count
// type: boolean
// default: false
// - in: query
// name: error_worker_count
// description: Indicates a request for error worker count
// type: boolean
// default: false
// responses:
// '200':
// description: Successfully retrieved the Vela metrics
Expand Down Expand Up @@ -375,14 +408,18 @@ func recordGauges(c *gin.Context) {

// add worker metrics
var (
buildLimit int64
activeWorkers int64
inactiveWorkers int64
buildLimit int64
activeWorkers int64
inactiveWorkers int64
idleWorkers int64
availableWorkers int64
busyWorkers int64
errorWorkers int64
)

// get worker metrics based on request query parameters
// worker_build_limit, active_worker_count, inactive_worker_count
if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount {
// worker_build_limit, active_worker_count, inactive_worker_count, idle_worker_count, available_worker_count, busy_worker_count, error_worker_count
if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount || q.IdleWorkerCount || q.AvailableWorkerCount || q.BusyWorkerCount || q.ErrorWorkerCount {
// send API call to capture the workers
workers, err := database.FromContext(c).ListWorkers()
if err != nil {
Expand All @@ -391,6 +428,9 @@ func recordGauges(c *gin.Context) {

// get the unix time from worker_active_interval ago
before := time.Now().UTC().Add(-c.Value("worker_active_interval").(time.Duration)).Unix()

// active, inactive counts
// idle, available, busy, error counts
for _, worker := range workers {
// check if the worker checked in within the last worker_active_interval
if worker.GetLastCheckedIn() >= before {
Expand All @@ -399,6 +439,20 @@ func recordGauges(c *gin.Context) {
} else {
inactiveWorkers++
}
// check if the worker checked in within the last worker_active_interval
if worker.GetLastCheckedIn() >= before {

switch worker.GetStatus() {
case constants.WorkerStatusIdle:
idleWorkers++
case constants.WorkerStatusAvailable:
availableWorkers++
case constants.WorkerStatusBusy:
busyWorkers++
case constants.WorkerStatusError:
errorWorkers++
}
}
}

// apply metrics based on request query parameters
Expand All @@ -416,5 +470,25 @@ func recordGauges(c *gin.Context) {
if q.InactiveWorkerCount {
totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers))
}

// idle_worker_count
if q.IdleWorkerCount {
totals.WithLabelValues("worker", "count", "idle").Set(float64(idleWorkers))
}

// available_worker_count
if q.AvailableWorkerCount {
totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers))
}

// busy_worker_count
if q.BusyWorkerCount {
totals.WithLabelValues("worker", "count", "busy").Set(float64(busyWorkers))
}

// error_worker_count
if q.ErrorWorkerCount {
totals.WithLabelValues("worker", "count", "error").Set(float64(errorWorkers))
}
}
}
25 changes: 25 additions & 0 deletions api/worker/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,31 @@ func UpdateWorker(c *gin.Context) {
w.SetActive(input.GetActive())
}

if input.RunningBuildIDs != nil {
// update runningBuildIDs if set
w.SetRunningBuildIDs(input.GetRunningBuildIDs())
}

if len(input.GetStatus()) > 0 {
// update status if set
w.SetStatus(input.GetStatus())
}

if input.GetLastStatusUpdateAt() > 0 {
// update lastStatusUpdateAt if set
w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt())
}

if input.GetLastBuildStartedAt() > 0 {
// update lastBuildStartedAt if set
w.SetLastBuildStartedAt(input.GetLastBuildStartedAt())
}

if input.GetLastBuildFinishedAt() > 0 {
// update lastBuildFinishedAt if set
w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt())
}

// send API call to update the worker
err = database.FromContext(c).UpdateWorker(w)
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions database/worker/create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) {

// ensure the mock expects the query
_mock.ExpectQuery(`INSERT INTO "workers"
("hostname","address","routes","active","last_checked_in","build_limit","id")
VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, 1).
("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_started_at","last_build_finished_at","last_checked_in","build_limit","id")
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12) RETURNING "id"`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1).
WillReturnRows(_rows)

_sqlite := testSqlite(t)
Expand Down
4 changes: 2 additions & 2 deletions database/worker/get_hostname_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ func TestWorker_Engine_GetWorkerForName(t *testing.T) {

// create expected result in mock
_rows := sqlmock.NewRows(
[]string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}).
AddRow(1, "worker_0", "localhost", nil, true, 0, 0)
[]string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}).
AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0)

// ensure the mock expects the query
_mock.ExpectQuery(`SELECT * FROM "workers" WHERE hostname = $1 LIMIT 1`).WithArgs("worker_0").WillReturnRows(_rows)
Expand Down
6 changes: 3 additions & 3 deletions database/worker/list_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,9 @@ func TestWorker_Engine_ListWorkers(t *testing.T) {

// create expected result in mock
_rows = sqlmock.NewRows(
[]string{"id", "hostname", "address", "routes", "active", "last_checked_in", "build_limit"}).
AddRow(1, "worker_0", "localhost", nil, true, 0, 0).
AddRow(2, "worker_1", "localhost", nil, true, 0, 0)
[]string{"id", "hostname", "address", "routes", "active", "status", "last_status_update_at", "running_build_ids", "last_build_started_at", "last_build_finished_at", "last_checked_in", "build_limit"}).
AddRow(1, "worker_0", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0).
AddRow(2, "worker_1", "localhost", nil, true, nil, 0, nil, 0, 0, 0, 0)

// ensure the mock expects the query
_mock.ExpectQuery(`SELECT * FROM "workers"`).WillReturnRows(_rows)
Expand Down
39 changes: 24 additions & 15 deletions database/worker/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,38 @@ const (
CREATE TABLE
IF NOT EXISTS
workers (
id SERIAL PRIMARY KEY,
hostname VARCHAR(250),
address VARCHAR(250),
routes VARCHAR(1000),
active BOOLEAN,
last_checked_in INTEGER,
build_limit INTEGER,
id SERIAL PRIMARY KEY,
hostname VARCHAR(250),
address VARCHAR(250),
routes VARCHAR(1000),
active BOOLEAN,
status VARCHAR(50),
last_status_update_at INTEGER,
running_build_ids VARCHAR(500),
last_build_started_at INTEGER,
last_build_finished_at INTEGER,
last_checked_in INTEGER,
build_limit INTEGER,
UNIQUE(hostname)
);
`

// CreateSqliteTable represents a query to create the Sqlite workers table.
CreateSqliteTable = `
CREATE TABLE
IF NOT EXISTS
workers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
hostname TEXT,
address TEXT,
routes TEXT,
active BOOLEAN,
last_checked_in INTEGER,
build_limit INTEGER,
id INTEGER PRIMARY KEY AUTOINCREMENT,
hostname TEXT,
address TEXT,
routes TEXT,
active BOOLEAN,
status VARCHAR(50),
last_status_update_at INTEGER,
running_build_ids VARCHAR(500),
last_build_started_at INTEGER,
last_build_finished_at INTEGER,
last_checked_in INTEGER,
build_limit INTEGER,
UNIQUE(hostname)
);
`
Expand Down
6 changes: 3 additions & 3 deletions database/worker/update_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ func TestWorker_Engine_UpdateWorker(t *testing.T) {

// ensure the mock expects the query
_mock.ExpectExec(`UPDATE "workers"
SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"last_checked_in"=$5,"build_limit"=$6
WHERE "id" = $7`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, 1).
SET "hostname"=$1,"address"=$2,"routes"=$3,"active"=$4,"status"=$5,"last_status_update_at"=$6,"running_build_ids"=$7,"last_build_started_at"=$8,"last_build_finished_at"=$9,"last_checked_in"=$10,"build_limit"=$11
WHERE "id" = $12`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, nil, 1).
WillReturnResult(sqlmock.NewResult(1, 1))

_sqlite := testSqlite(t)
Expand Down
19 changes: 12 additions & 7 deletions database/worker/worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,12 +170,17 @@ func testSqlite(t *testing.T) *engine {
// Worker type with all fields set to their zero values.
func testWorker() *library.Worker {
return &library.Worker{
ID: new(int64),
Hostname: new(string),
Address: new(string),
Routes: new([]string),
Active: new(bool),
BuildLimit: new(int64),
LastCheckedIn: new(int64),
ID: new(int64),
Hostname: new(string),
Address: new(string),
Routes: new([]string),
Active: new(bool),
Status: new(string),
LastStatusUpdateAt: new(int64),
RunningBuildIDs: new([]string),
LastBuildStartedAt: new(int64),
LastBuildFinishedAt: new(int64),
LastCheckedIn: new(int64),
BuildLimit: new(int64),
}
}
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/drone/envsubst v1.0.3
github.com/gin-gonic/gin v1.9.0
github.com/go-playground/assert/v2 v2.2.0
github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c
github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af
github.com/golang-jwt/jwt/v5 v5.0.0
github.com/google/go-cmp v0.5.9
github.com/google/go-github/v52 v52.0.0
Expand Down Expand Up @@ -88,12 +88,12 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.0.9 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lib/pq v1.10.8 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/mattn/go-colorable v0.1.8 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
github.com/microcosm-cc/bluemonday v1.0.24 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
Expand All @@ -115,8 +115,8 @@ require (
github.com/yuin/gopher-lua v1.1.0 // indirect
golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
golang.org/x/crypto v0.8.0 // indirect
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
20 changes: 10 additions & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU=
github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s=
github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c h1:eAApIK5e5MxFF8RzZAFsvTSdwq/AzdUrdhJHOGQ0ILc=
github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs=
github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af h1:Ixsa6Ha0j9Edq4v3IooDgyUoGSp08fk9FgrYKuZSML8=
github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk=
github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
Expand Down Expand Up @@ -290,8 +290,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lib/pq v1.10.8 h1:3fdt97i/cwSU83+E0hZTC/Xpc9mTZxc6UWSCRcSbxiE=
github.com/lib/pq v1.10.8/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand All @@ -306,8 +306,8 @@ github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJK
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY=
github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4=
github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
Expand Down Expand Up @@ -481,8 +481,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -552,8 +552,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
Expand Down
Loading

0 comments on commit 4e5d484

Please sign in to comment.