Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(config): Make MaximumNumberOfResults and MaximumNumberOfEvents configurable #476

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -299,12 +299,14 @@ Here are some examples of conditions you can use:


### Storage
| Parameter | Description | Default |
|:------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------|:-----------|
| `storage` | Storage configuration | `{}` |
| `storage.path` | Path to persist the data in. Only supported for types `sqlite` and `postgres`. | `""` |
| `storage.type` | Type of storage. Valid types: `memory`, `sqlite`, `postgres`. | `"memory"` |
| `storage.caching` | Whether to use write-through caching. Improves loading time for large dashboards. <br />Only supported if `storage.type` is `sqlite` or `postgres` | `false` |
| Parameter | Description | Default |
|:------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------|:-----------|
| `storage` | Storage configuration | `{}` |
| `storage.path` | Path to persist the data in. Only supported for types `sqlite` and `postgres`. | `""` |
| `storage.type` | Type of storage. Valid types: `memory`, `sqlite`, `postgres`. | `"memory"` |
| `storage.caching` | Whether to use write-through caching. Improves loading time for large dashboards. <br />Only supported if `storage.type` is `sqlite` or `postgres` | `false` |
| `storage.maximum-number-of-results` | The maximum number of results that an endpoint can have | `100` |
| `storage.maximum-number-of-events` | The maximum number of events that an endpoint can have | `50` |

The results for each endpoint health check as well as the data for uptime and the past events must be persisted
so that they can be displayed on the dashboard. These parameters allow you to configure the storage in question.
Expand Down
2 changes: 1 addition & 1 deletion api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,6 @@ func (a *API) createRouter(cfg *config.Config) *fiber.App {
}
}
protectedAPIRouter.Get("/v1/endpoints/statuses", EndpointStatuses(cfg))
protectedAPIRouter.Get("/v1/endpoints/:key/statuses", EndpointStatus)
protectedAPIRouter.Get("/v1/endpoints/:key/statuses", EndpointStatus(cfg))
return app
}
42 changes: 22 additions & 20 deletions api/endpoint_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
// Due to how intensive this operation can be on the storage, this function leverages a cache.
func EndpointStatuses(cfg *config.Config) fiber.Handler {
return func(c *fiber.Ctx) error {
page, pageSize := extractPageAndPageSizeFromRequest(c)
page, pageSize := extractPageAndPageSizeFromRequest(c, cfg.Storage.MaximumNumberOfResults)
value, exists := cache.Get(fmt.Sprintf("endpoint-status-%d-%d", page, pageSize))
var data []byte
if !exists {
Expand Down Expand Up @@ -83,25 +83,27 @@ func getEndpointStatusesFromRemoteInstances(remoteConfig *remote.Config) ([]*cor
}

// EndpointStatus retrieves a single core.EndpointStatus by group and endpoint name
func EndpointStatus(c *fiber.Ctx) error {
page, pageSize := extractPageAndPageSizeFromRequest(c)
endpointStatus, err := store.Get().GetEndpointStatusByKey(c.Params("key"), paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, common.MaximumNumberOfEvents))
if err != nil {
if err == common.ErrEndpointNotFound {
return c.Status(404).SendString(err.Error())
func EndpointStatus(cfg *config.Config) fiber.Handler {
return func(c *fiber.Ctx) error {
page, pageSize := extractPageAndPageSizeFromRequest(c, cfg.Storage.MaximumNumberOfResults)
endpointStatus, err := store.Get().GetEndpointStatusByKey(c.Params("key"), paging.NewEndpointStatusParams().WithResults(page, pageSize).WithEvents(1, cfg.Storage.MaximumNumberOfEvents))
if err != nil {
if err == common.ErrEndpointNotFound {
return c.Status(404).SendString(err.Error())
}
log.Printf("[api][EndpointStatus] Failed to retrieve endpoint status: %s", err.Error())
return c.Status(500).SendString(err.Error())
}
log.Printf("[api][EndpointStatus] Failed to retrieve endpoint status: %s", err.Error())
return c.Status(500).SendString(err.Error())
}
if endpointStatus == nil { // XXX: is this check necessary?
log.Printf("[api][EndpointStatus] Endpoint with key=%s not found", c.Params("key"))
return c.Status(404).SendString("not found")
}
output, err := json.Marshal(endpointStatus)
if err != nil {
log.Printf("[api][EndpointStatus] Unable to marshal object to JSON: %s", err.Error())
return c.Status(500).SendString("unable to marshal object to JSON")
if endpointStatus == nil { // XXX: is this check necessary?
log.Printf("[api][EndpointStatus] Endpoint with key=%s not found", c.Params("key"))
return c.Status(404).SendString("not found")
}
output, err := json.Marshal(endpointStatus)
if err != nil {
log.Printf("[api][EndpointStatus] Unable to marshal object to JSON: %s", err.Error())
return c.Status(500).SendString("unable to marshal object to JSON")
}
c.Set("Content-Type", "application/json")
return c.Status(200).Send(output)
}
c.Set("Content-Type", "application/json")
return c.Status(200).Send(output)
}
13 changes: 12 additions & 1 deletion api/endpoint_status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/TwiN/gatus/v5/core"
"github.com/TwiN/gatus/v5/storage/store"
"github.com/TwiN/gatus/v5/watchdog"
"github.com/TwiN/gatus/v5/storage"
)

var (
Expand Down Expand Up @@ -95,6 +96,10 @@ func TestEndpointStatus(t *testing.T) {
Group: "core",
},
},
Storage: &storage.Config{
MaximumNumberOfResults: storage.DefaultMaximumNumberOfResults,
MaximumNumberOfEvents: storage.DefaultMaximumNumberOfEvents,
},
}
watchdog.UpdateEndpointStatuses(cfg.Endpoints[0], &core.Result{Success: true, Duration: time.Millisecond, Timestamp: time.Now()})
watchdog.UpdateEndpointStatuses(cfg.Endpoints[1], &core.Result{Success: false, Duration: time.Second, Timestamp: time.Now()})
Expand Down Expand Up @@ -156,7 +161,13 @@ func TestEndpointStatuses(t *testing.T) {
// Can't be bothered dealing with timezone issues on the worker that runs the automated tests
firstResult.Timestamp = time.Time{}
secondResult.Timestamp = time.Time{}
api := New(&config.Config{Metrics: true})
api := New(&config.Config{
Metrics: true,
Storage: &storage.Config{
MaximumNumberOfResults: storage.DefaultMaximumNumberOfResults,
MaximumNumberOfEvents: storage.DefaultMaximumNumberOfEvents,
},
})
router := api.Router()
type Scenario struct {
Name string
Expand Down
10 changes: 3 additions & 7 deletions api/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package api
import (
"strconv"

"github.com/TwiN/gatus/v5/storage/store/common"
"github.com/gofiber/fiber/v2"
)

Expand All @@ -13,12 +12,9 @@ const (

// DefaultPageSize is the default page siZE to use if none is specified or an invalid value is provided
DefaultPageSize = 20

// MaximumPageSize is the maximum page size allowed
MaximumPageSize = common.MaximumNumberOfResults
)

func extractPageAndPageSizeFromRequest(c *fiber.Ctx) (page, pageSize int) {
func extractPageAndPageSizeFromRequest(c *fiber.Ctx, maximumNumberOfResults int) (page, pageSize int) {
var err error
if pageParameter := c.Query("page"); len(pageParameter) == 0 {
page = DefaultPage
Expand All @@ -38,8 +34,8 @@ func extractPageAndPageSizeFromRequest(c *fiber.Ctx) (page, pageSize int) {
if err != nil {
pageSize = DefaultPageSize
}
if pageSize > MaximumPageSize {
pageSize = MaximumPageSize
if pageSize > maximumNumberOfResults {
pageSize = maximumNumberOfResults
} else if pageSize < 1 {
pageSize = DefaultPageSize
}
Expand Down
68 changes: 38 additions & 30 deletions api/util_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,54 +4,62 @@ import (
"fmt"
"testing"

"github.com/TwiN/gatus/v5/storage"
"github.com/gofiber/fiber/v2"
"github.com/valyala/fasthttp"
)

func TestExtractPageAndPageSizeFromRequest(t *testing.T) {
type Scenario struct {
Name string
Page string
PageSize string
ExpectedPage int
ExpectedPageSize int
Name string
Page string
PageSize string
ExpectedPage int
ExpectedPageSize int
MaximumNumberOfResults int
}
scenarios := []Scenario{
{
Page: "1",
PageSize: "20",
ExpectedPage: 1,
ExpectedPageSize: 20,
Page: "1",
PageSize: "20",
ExpectedPage: 1,
ExpectedPageSize: 20,
MaximumNumberOfResults: 20,
},
{
Page: "2",
PageSize: "10",
ExpectedPage: 2,
ExpectedPageSize: 10,
Page: "2",
PageSize: "10",
ExpectedPage: 2,
ExpectedPageSize: 10,
MaximumNumberOfResults: 40,
},
{
Page: "2",
PageSize: "10",
ExpectedPage: 2,
ExpectedPageSize: 10,
Page: "2",
PageSize: "10",
ExpectedPage: 2,
ExpectedPageSize: 10,
MaximumNumberOfResults: 200,
},
{
Page: "1",
PageSize: "999999",
ExpectedPage: 1,
ExpectedPageSize: MaximumPageSize,
Page: "1",
PageSize: "999999",
ExpectedPage: 1,
ExpectedPageSize: storage.DefaultMaximumNumberOfResults,
MaximumNumberOfResults: 100,
},
{
Page: "-1",
PageSize: "-1",
ExpectedPage: DefaultPage,
ExpectedPageSize: DefaultPageSize,
Page: "-1",
PageSize: "-1",
ExpectedPage: DefaultPage,
ExpectedPageSize: DefaultPageSize,
MaximumNumberOfResults: 20,
},
{
Page: "invalid",
PageSize: "invalid",
ExpectedPage: DefaultPage,
ExpectedPageSize: DefaultPageSize,
Page: "invalid",
PageSize: "invalid",
ExpectedPage: DefaultPage,
ExpectedPageSize: DefaultPageSize,
MaximumNumberOfResults: 100,
},
}
for _, scenario := range scenarios {
Expand All @@ -61,7 +69,7 @@ func TestExtractPageAndPageSizeFromRequest(t *testing.T) {
c := app.AcquireCtx(&fasthttp.RequestCtx{})
defer app.ReleaseCtx(c)
c.Request().SetRequestURI(fmt.Sprintf("/api/v1/statuses?page=%s&pageSize=%s", scenario.Page, scenario.PageSize))
actualPage, actualPageSize := extractPageAndPageSizeFromRequest(c)
actualPage, actualPageSize := extractPageAndPageSizeFromRequest(c, scenario.MaximumNumberOfResults)
if actualPage != scenario.ExpectedPage {
t.Errorf("expected %d, got %d", scenario.ExpectedPage, actualPage)
}
Expand Down
2 changes: 2 additions & 0 deletions config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,5 @@ endpoints:
interval: 1h
conditions:
- "[DOMAIN_EXPIRATION] > 720h"
storage:
maximum-number-of-results: 1000
5 changes: 4 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,7 @@ func parseAndValidateConfigBytes(yamlBytes []byte) (config *Config, err error) {
if err := validateConnectivityConfig(config); err != nil {
return nil, err
}
config.UI.MaximumNumberOfResults = config.Storage.MaximumNumberOfResults
}
return
}
Expand All @@ -282,7 +283,9 @@ func validateRemoteConfig(config *Config) error {
func validateStorageConfig(config *Config) error {
if config.Storage == nil {
config.Storage = &storage.Config{
Type: storage.TypeMemory,
Type: storage.TypeMemory,
MaximumNumberOfResults: storage.DefaultMaximumNumberOfResults,
MaximumNumberOfEvents: storage.DefaultMaximumNumberOfEvents,
}
} else {
if err := config.Storage.ValidateAndSetDefaults(); err != nil {
Expand Down
5 changes: 5 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,8 @@ func TestParseAndValidateConfigBytes(t *testing.T) {
storage:
type: sqlite
path: %s
maximum-number-of-results: 10
maximum-number-of-events: 5
maintenance:
enabled: true
start: 00:00
Expand Down Expand Up @@ -351,6 +353,9 @@ endpoints:
if config.Storage == nil || config.Storage.Path != file || config.Storage.Type != storage.TypeSQLite {
t.Error("expected storage to be set to sqlite, got", config.Storage)
}
if config.Storage == nil || config.Storage.MaximumNumberOfResults != 10 || config.Storage.MaximumNumberOfEvents != 5 {
t.Error("expected MaximumNumberOfResults and MaximumNumberOfEvents to be set to 10 and 5, got", config.Storage.MaximumNumberOfResults, config.Storage.MaximumNumberOfEvents)
}
if config.UI == nil || config.UI.Title != "T" || config.UI.Header != "H" || config.UI.Link != "https://example.org" || len(config.UI.Buttons) != 2 || config.UI.Buttons[0].Name != "Home" || config.UI.Buttons[0].Link != "https://example.org" || config.UI.Buttons[1].Name != "Status page" || config.UI.Buttons[1].Link != "https://status.example.org" {
t.Error("expected ui to be set to T, H, https://example.org, 2 buttons, Home and Status page, got", config.UI)
}
Expand Down
14 changes: 9 additions & 5 deletions config/ui/ui.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"errors"
"html/template"

"github.com/TwiN/gatus/v5/storage"
static "github.com/TwiN/gatus/v5/web"
)

Expand All @@ -28,6 +29,8 @@ type Config struct {
Logo string `yaml:"logo,omitempty"` // Logo to display on the page
Link string `yaml:"link,omitempty"` // Link to open when clicking on the logo
Buttons []Button `yaml:"buttons,omitempty"` // Buttons to display below the header

MaximumNumberOfResults int // MaximumNumberOfResults to display on the page, it's not configurable because we're passing it from the storage config
}

// Button is the configuration for a button on the UI
Expand All @@ -47,11 +50,12 @@ func (btn *Button) Validate() error {
// GetDefaultConfig returns a Config struct with the default values
func GetDefaultConfig() *Config {
return &Config{
Title: defaultTitle,
Description: defaultDescription,
Header: defaultHeader,
Logo: defaultLogo,
Link: defaultLink,
Title: defaultTitle,
Description: defaultDescription,
Header: defaultHeader,
Logo: defaultLogo,
Link: defaultLink,
MaximumNumberOfResults: storage.DefaultMaximumNumberOfResults,
}
}

Expand Down
17 changes: 17 additions & 0 deletions storage/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ import (
"errors"
)

const (
DefaultMaximumNumberOfResults = 100
DefaultMaximumNumberOfEvents = 50
)

var (
ErrSQLStorageRequiresPath = errors.New("sql storage requires a non-empty path to be defined")
ErrMemoryStorageDoesNotSupportPath = errors.New("memory storage does not support persistence, use sqlite if you want persistence on file")
Expand All @@ -25,6 +30,12 @@ type Config struct {
// as they happen, also known as the write-through caching strategy.
// Does not apply if Config.Type is not TypePostgres or TypeSQLite.
Caching bool `yaml:"caching,omitempty"`

// MaximumNumberOfResults is the maximum number of results that an endpoint can have
MaximumNumberOfResults int `yaml:"maximum-number-of-results,omitempty"`

// MaximumNumberOfEvents is the maximum number of events that an endpoint can have
MaximumNumberOfEvents int `yaml:"maximum-number-of-events,omitempty"`
}

// ValidateAndSetDefaults validates the configuration and sets the default values (if applicable)
Expand All @@ -38,5 +49,11 @@ func (c *Config) ValidateAndSetDefaults() error {
if c.Type == TypeMemory && len(c.Path) > 0 {
return ErrMemoryStorageDoesNotSupportPath
}
if c.MaximumNumberOfResults <= 0 {
c.MaximumNumberOfResults = DefaultMaximumNumberOfResults
}
if c.MaximumNumberOfEvents <= 0 {
c.MaximumNumberOfEvents = DefaultMaximumNumberOfEvents
}
return nil
}
9 changes: 0 additions & 9 deletions storage/store/common/limits.go

This file was deleted.

Loading