Async Healthcheck (#301)

* Async health check

* Tests adaptation

* Make health configurable externally

* Install health later

* Export ConvertStatus function

* Fix health settings validation

* Attach logger to health

* Address PR comments

* Configuration per indicator and refactoring

* Add health status listener

* Minor tweaks

* Fix indicator interval type

* Fix tests

* Remove unused import

* Extract indicator configuration and address PR comments

* Add error to panic

* Address PR comments

* minor fix

* rename storage indicator

Gopkg.toml

@@ -108,4 +108,8 @@
 # Refer to issue https://github.com/golang/dep/issues/1799
 [[override]]
 name = "gopkg.in/fsnotify.v1"
-source = "https://github.com/fsnotify/fsnotify.git"
+source = "https://github.com/fsnotify/fsnotify.git"
+
+[[constraint]]
+  name = "github.com/InVisionApp/go-health"
+  version = "v2.1.0"

api/healthcheck/healthcheck_controller.go

@@ -17,32 +17,36 @@
 package healthcheck
 
 import (
+	h "github.com/InVisionApp/go-health"
+	"github.com/Peripli/service-manager/pkg/health"
 	"github.com/Peripli/service-manager/pkg/util"
 	"net/http"
 
-	"github.com/Peripli/service-manager/pkg/health"
 	"github.com/Peripli/service-manager/pkg/log"
 	"github.com/Peripli/service-manager/pkg/web"
 )
 
 // controller platform controller
 type controller struct {
-	indicator health.Indicator
+	health     h.IHealth
+	thresholds map[string]int64
 }
 
-// NewController returns a new healthcheck controller with the given indicators and aggregation policy
-func NewController(indicators []health.Indicator, aggregator health.AggregationPolicy) web.Controller {
+// NewController returns a new healthcheck controller with the given health and thresholds
+func NewController(health h.IHealth, thresholds map[string]int64) web.Controller {
 	return &controller{
-		indicator: newCompositeIndicator(indicators, aggregator),
+		health:     health,
+		thresholds: thresholds,
 	}
 }
 
 // healthCheck handler for GET /v1/monitor/health
 func (c *controller) healthCheck(r *web.Request) (*web.Response, error) {
 	ctx := r.Context()
 	logger := log.C(ctx)
-	logger.Debugf("Performing health check with %s...", c.indicator.Name())
-	healthResult := c.indicator.Health()
+	logger.Debugf("Performing health check...")
+	healthState, _, _ := c.health.State()
+	healthResult := c.aggregate(healthState)
 	var status int
 	if healthResult.Status == health.StatusUp {
 		status = http.StatusOK
@@ -51,3 +55,33 @@ func (c *controller) healthCheck(r *web.Request) (*web.Response, error) {
 	}
 	return util.NewJSONResponse(status, healthResult)
 }
+
+func (c *controller) aggregate(overallState map[string]h.State) *health.Health {
+	if len(overallState) == 0 {
+		return health.New().WithStatus(health.StatusUp)
+	}
+	overallStatus := health.StatusUp
+	for name, state := range overallState {
+		if state.Fatal && state.ContiguousFailures >= c.thresholds[name] {
+			overallStatus = health.StatusDown
+			break
+		}
+	}
+	details := make(map[string]interface{})
+	for name, state := range overallState {
+		state.Status = convertStatus(state.Status)
+		details[name] = state
+	}
+	return health.New().WithStatus(overallStatus).WithDetails(details)
+}
+
+func convertStatus(status string) string {
+	switch status {
+	case "ok":
+		return string(health.StatusUp)
+	case "failed":
+		return string(health.StatusDown)
+	default:
+		return string(health.StatusUnknown)
+	}
+}