Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add http failure reason metric #1077 #1139

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Add http failure reason metric #1077 #1139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7d22e65
http failure reason #1077
Oct 19, 2023
20c9214
add generic urlparse error
Oct 23, 2023
658a58e
handle counter metrics in test helper checkRegistryResults
Oct 23, 2023
c6d47d7
fix TestFailIfNotSSLLogMsg error reason
Oct 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions prober/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"compress/gzip"
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -296,6 +297,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
Name: "probe_http_last_modified_timestamp_seconds",
Help: "Returns the Last-Modified HTTP response header in unixtime",
})
probeFailureCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "probe_http_failures_total",
Help: "Counts number of probe http failures by reason",
}, []string{"reason"})
)

registry.MustRegister(durationGaugeVec)
Expand All @@ -306,6 +311,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
registry.MustRegister(statusCodeGauge)
registry.MustRegister(probeHTTPVersionGauge)
registry.MustRegister(probeFailedDueToRegex)
registry.MustRegister(probeFailureCounter)

httpConfig := module.HTTP

Expand All @@ -316,6 +322,12 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
targetURL, err := url.Parse(target)
if err != nil {
level.Error(logger).Log("msg", "Could not parse target URL", "err", err)
var ue *url.Error
if errors.As(err, &ue) {
probeFailureCounter.WithLabelValues(labelFromUrlError("urlparse", ue)).Inc()
} else {
probeFailureCounter.WithLabelValues("urlparse_error").Inc()
}
return false
}

Expand All @@ -328,6 +340,21 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
ip, lookupTime, err = chooseProtocol(ctx, module.HTTP.IPProtocol, module.HTTP.IPProtocolFallback, targetHost, registry, logger)
durationGaugeVec.WithLabelValues("resolve").Add(lookupTime)
if err != nil {
var de *net.DNSError
if errors.As(err, &de) {
switch {
case de.IsNotFound:
probeFailureCounter.WithLabelValues("dns_not_found").Inc()
case de.IsTemporary:
probeFailureCounter.WithLabelValues("dns_temporary_failure").Inc()
case de.IsTimeout:
probeFailureCounter.WithLabelValues("dns_timeout").Inc()
default:
probeFailureCounter.WithLabelValues("dns_error").Inc()
}
} else {
probeFailureCounter.WithLabelValues("dns_error").Inc()
}
level.Error(logger).Log("msg", "Error resolving address", "err", err)
return false
}
Expand All @@ -351,19 +378,22 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
client, err := pconfig.NewClientFromConfig(httpClientConfig, "http_probe", pconfig.WithKeepAlivesDisabled())
if err != nil {
level.Error(logger).Log("msg", "Error generating HTTP client", "err", err)
probeFailureCounter.WithLabelValues("client_error").Inc()
return false
}

httpClientConfig.TLSConfig.ServerName = ""
noServerName, err := pconfig.NewRoundTripperFromConfig(httpClientConfig, "http_probe", pconfig.WithKeepAlivesDisabled())
if err != nil {
level.Error(logger).Log("msg", "Error generating HTTP client without ServerName", "err", err)
probeFailureCounter.WithLabelValues("client_error").Inc()
return false
}

jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
if err != nil {
level.Error(logger).Log("msg", "Error generating cookiejar", "err", err)
probeFailureCounter.WithLabelValues("cookiejar_error").Inc()
return false
}
client.Jar = jar
Expand Down Expand Up @@ -414,6 +444,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
body_file, err := os.Open(httpConfig.BodyFile)
if err != nil {
level.Error(logger).Log("msg", "Error creating request", "err", err)
probeFailureCounter.WithLabelValues("request_creation_error").Inc()
return
}
defer body_file.Close()
Expand All @@ -423,6 +454,12 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
request, err := http.NewRequest(httpConfig.Method, targetURL.String(), body)
if err != nil {
level.Error(logger).Log("msg", "Error creating request", "err", err)
var ue *url.Error
if errors.As(err, &ue) {
probeFailureCounter.WithLabelValues(labelFromUrlError("request_creation", ue)).Inc()
} else {
probeFailureCounter.WithLabelValues("request_creation_error").Inc()
}
return
}
request.Host = origHost
Expand Down Expand Up @@ -468,6 +505,22 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
resp = &http.Response{}
if err != nil {
level.Error(logger).Log("msg", "Error for HTTP request", "err", err)
var authorityError x509.UnknownAuthorityError
var hostnameError x509.HostnameError
var certInvalidError x509.CertificateInvalidError
var ue *url.Error
switch {
case errors.As(err, &authorityError):
probeFailureCounter.WithLabelValues("request_certificate_unknown_authority").Inc()
case errors.As(err, &hostnameError):
probeFailureCounter.WithLabelValues("request_certificate_hostname_mismatch").Inc()
case errors.As(err, &certInvalidError):
probeFailureCounter.WithLabelValues("request_certificate_invalid").Inc()
case errors.As(err, &ue):
probeFailureCounter.WithLabelValues(labelFromUrlError("request", ue)).Inc()
default:
probeFailureCounter.WithLabelValues("request_error").Inc()
}
}
} else {
requestErrored := (err != nil)
Expand Down Expand Up @@ -506,6 +559,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
dec, err := getDecompressionReader(httpConfig.Compression, resp.Body)
if err != nil {
level.Info(logger).Log("msg", "Failed to get decompressor for HTTP response body", "err", err)
probeFailureCounter.WithLabelValues("decompression_error").Inc()
success = false
} else if dec != nil {
// Since we are replacing the original resp.Body with the decoder, we need to make sure
Expand Down Expand Up @@ -547,6 +601,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
_, err = io.Copy(io.Discard, byteCounter)
if err != nil {
level.Info(logger).Log("msg", "Failed to read HTTP response body", "err", err)
probeFailureCounter.WithLabelValues("read_body_error").Inc()
success = false
}

Expand Down Expand Up @@ -586,6 +641,7 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
}
if !found {
level.Error(logger).Log("msg", "Invalid HTTP version number", "version", resp.Proto)
probeFailureCounter.WithLabelValues("invalid_http_version").Inc()
success = false
}
}
Expand Down Expand Up @@ -645,10 +701,12 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
probeSSLLastInformation.WithLabelValues(getFingerprint(resp.TLS), getSubject(resp.TLS), getIssuer(resp.TLS), getDNSNames(resp.TLS)).Set(1)
if httpConfig.FailIfSSL {
level.Error(logger).Log("msg", "Final request was over SSL")
probeFailureCounter.WithLabelValues("final_request_ssl").Inc()
success = false
}
} else if httpConfig.FailIfNotSSL && success {
level.Error(logger).Log("msg", "Final request was not over SSL")
probeFailureCounter.WithLabelValues("final_request_not_ssl").Inc()
success = false
}

Expand Down Expand Up @@ -677,3 +735,15 @@ func getDecompressionReader(algorithm string, origBody io.ReadCloser) (io.ReadCl
return nil, errors.New("unsupported compression algorithm")
}
}

func labelFromUrlError(prefix string, err *url.Error) string {
op := strings.ToLower(err.Op)
switch {
case err.Timeout():
return fmt.Sprintf("%s_%s_timeout", prefix, op)
case err.Temporary():
return fmt.Sprintf("%s_%s_temporary_failure", prefix, op)
default:
return fmt.Sprintf("%s_%s_error", prefix, op)
}
}
Loading