Batching the probes to spread the workload and more exact time readings

.gitignore

@@ -1,2 +1,4 @@
 /board
 /board.yaml
+bin/*
+tmp/*

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:latest  AS build-env
+FROM golang:1.16-alpine AS build-env
 
 # Dependencies
 WORKDIR /build

Makefile

@@ -0,0 +1,39 @@
+BINDIR ?= $(CURDIR)/bin
+TMPDIR ?= $(CURDIR)/tmp
+ARCH   ?= amd64
+
+help:  ## display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n\nTargets:\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
+
+.PHONY: help build image all clean dev
+
+test: ## test board
+	go test ./...
+
+dev: ## live reload development
+	gin --path . --appPort 8080 --all --immediate --bin tmp/board run
+
+build: ## build board
+	mkdir -p $(BINDIR)
+	CGO_ENABLED=0 go build -o ./bin/board
+
+verify: test build ## tests and builds board
+
+image: ## build docker image
+	docker build -t lesterpig/board:latest .
+
+clean: ## clean up created files
+	rm -rf \
+		$(BINDIR) \
+		$(TMPDIR)
+
+all: clean test build image ## runs test, build and image
+
+test-coverage: ## Generate test coverage report
+	mkdir -p $(TMPDIR)
+	go test ./... --coverprofile $(TMPDIR)/outfile
+	go tool cover -html=$(TMPDIR)/outfile
+
+lint: ## Generate static analysis report
+	goreportcard-cli -v
+	golint ./...

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/gobuffalo/envy v1.8.1
 	github.com/miekg/dns v1.1.13
 	github.com/mitchellh/mapstructure v1.1.2
+	github.com/prometheus/client_golang v0.9.3
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/viper v1.4.0
 	github.com/stretchr/testify v1.4.0

go.sum

@@ -10,6 +10,7 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -40,6 +41,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -66,6 +68,7 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.13 h1:x7DQtkU0cedzeS8TD36tT/w1Hm4rDtfCaYYAHE7TTBI=
 github.com/miekg/dns v1.1.13/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -80,12 +83,16 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

main.go

@@ -11,6 +11,7 @@ import (
 
 	rice "github.com/GeertJohan/go.rice"
 	"github.com/gobuffalo/envy"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 )
 
@@ -43,6 +44,8 @@ func main() {
 		_, _ = w.Write(data)
 	})
 
+	http.Handle("/metrics", promhttp.Handler())
+
 	go manager.ProbeLoop(time.Duration(int64(interval)) * time.Minute)
 	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(*port), nil))
 }

manager.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"math"
 	"time"
 
 	"github.com/Lesterpig/board/probe"
@@ -38,8 +39,17 @@ func (manager *Manager) ProbeAll() {
 
 	manager.LastUpdate = time.Now()
 
+	i := 0.0
+
 	for category, services := range manager.Services {
 		for _, service := range services {
+			// Batching the probes to spread the workload
+			if math.Mod(i, 8) == 0 {
+				log.Debug("Waiting")
+				time.Sleep(2 * time.Second)
+				i = 0
+			}
+			i++
 			go func(category string, service *Service) {
 				prevStatus := service.Status
 				service.Status, service.Message = service.Prober.Probe()

probe/http.go

@@ -2,8 +2,10 @@ package probe
 
 import (
 	"crypto/tls"
+	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httptrace"
 	"regexp"
 	"strconv"
 	"time"
@@ -38,9 +40,11 @@ func (h *HTTP) Init(c Config) error {
 	}
 
 	/* #nosec G402 */
-	tr := &http.Transport{
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: !opts.VerifyCertificate},
-	}
+	tr := http.DefaultTransport.(*http.Transport).Clone()
+	tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	// Disable keep alive to get more consistent measurements
+	tr.DisableKeepAlives = true
+	tr.ResponseHeaderTimeout = c.Fatal
 
 	h.client = &http.Client{
 		Timeout:   c.Fatal,
@@ -55,17 +59,39 @@ func (h *HTTP) Init(c Config) error {
 // If the operation succeeds, the message will be the duration of the HTTP request in ms.
 // Otherwise, an error message is returned.
 func (h *HTTP) Probe() (status Status, message string) {
-	start := time.Now()
-	res, err := h.client.Get(h.Target)
-	duration := time.Since(start)
+
+	req, _ := http.NewRequest("GET", h.Target, nil)
+
+	var server, start time.Time
+	var initDuration, serverDuration, totalDuration time.Duration
+
+	trace := &httptrace.ClientTrace{
+
+		TLSHandshakeDone: func(cs tls.ConnectionState, err error) {
+			server = time.Now()
+			initDuration = time.Since(start)
+		},
+	}
+
+	req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
+	// Starting timer
+	start = time.Now()
+	// Set server time before request. This may be reset in TLSHandshakeDone if a TLSHandshake is preformed
+	server = start
+
+	res, err := h.client.Transport.RoundTrip(req)
+	totalDuration = time.Since(start)
+	serverDuration = time.Since(server)
+
+	if res != nil {
+		defer res.Body.Close() // MUST CLOSED THIS
+	}
 
 	if err != nil {
 		return StatusError, defaultConnectErrorMsg
 	}
 
-	defer func() { _ = res.Body.Close() }()
-
-	if res.StatusCode != 200 {
+	if !(res.StatusCode >= 200 && res.StatusCode <= 399) {
 		return StatusError, strconv.Itoa(res.StatusCode)
 	}
 
@@ -74,5 +100,20 @@ func (h *HTTP) Probe() (status Status, message string) {
 		return StatusError, "Unexpected result"
 	}
 
-	return EvaluateDuration(duration, h.Warning)
+	return AdvancedEvaluateDuration(initDuration, serverDuration, totalDuration, h.Warning)
+}
+
+// AdvancedEvaluateDuration is a shortcut for warning duration checks.
+// It returns a message containing the duration, and a OK or a WARNING status
+// depending on the provided warning duration.
+func AdvancedEvaluateDuration(initDuration, serverDuration, duration time.Duration, warning time.Duration) (status Status, message string) {
+	if duration >= warning {
+		status = StatusWarning
+	} else {
+		status = StatusOK
+	}
+
+	message = fmt.Sprintf("%d ms / %d ms", initDuration.Nanoseconds()/1000000, serverDuration.Nanoseconds()/1000000)
+
+	return
 }

probe/probe.go

@@ -4,8 +4,12 @@ package probe
 import (
 	"fmt"
 	"time"
+
+	"github.com/sirupsen/logrus"
 )
 
+var log = logrus.StandardLogger()
+
 // Config holds probe configuration, submitted through Init methods.
 type Config struct {
 	Target  string