From 6969e164b5f03f155c89e0fa3abaa19d93582997 Mon Sep 17 00:00:00 2001 From: Oleksandr Novak Date: Mon, 4 Nov 2024 15:08:46 +0100 Subject: [PATCH] fix: improve error handling --- internal/metrics/metrics.go | 4 ++-- internal/monitor.go | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 52bf3bc..e01c014 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -24,8 +24,8 @@ func ReportNotSupported(namespace string, hpaName string) { errorsMetric.WithLabelValues(namespace, hpaName, "not_supported").Inc() } -func ReportBadHpaState(namespace string, volumeType string) { - errorsMetric.WithLabelValues(namespace, volumeType, "hpa_state").Inc() +func ReportBadHpaState(namespace string, hpaName string) { + errorsMetric.WithLabelValues(namespace, hpaName, "hpa_state").Inc() } func ReportCustomMetricError(namespace string, volumeType string) { errorsMetric.WithLabelValues(namespace, volumeType, "custom_metric").Inc() diff --git a/internal/monitor.go b/internal/monitor.go index 1fcd14f..6dda4f1 100644 --- a/internal/monitor.go +++ b/internal/monitor.go @@ -157,6 +157,9 @@ func requestMetricValuesFromSpec(ctx hpaScopedContext) (*[]bool, error) { } else { metricValues <- isZero } + } else if metric.Type == "" { + metrics.ReportBadHpaState(ctx.hpa.Namespace, ctx.hpa.Name) + ctx.logger.Error(nil, fmt.Sprintf("unexpected response: hpa returned metric definition with no type")) } else { metrics.ReportNotSupported(ctx.hpa.Namespace, ctx.hpa.Name) ctx.logger.Error(nil, fmt.Sprintf("not supported metric type '%s'", metric.Type)) @@ -203,6 +206,9 @@ func extractMetricValuesFromCurrentMetrics(hpa *autoscaling.HorizontalPodAutosca isZero = metric.Object.CurrentValue.IsZero() } else if metric.Type == "External" { isZero = metric.External.CurrentValue.IsZero() + } else if metric.Type == "" { + metrics.ReportBadHpaState(hpa.Namespace, hpa.Name) + return nil, fmt.Errorf("unexpected response: hpa returned metric with no type") } else { metrics.ReportNotSupported(hpa.Namespace, hpa.Name) return nil, fmt.Errorf("not supported metric type %q", metric.Type) @@ -262,12 +268,16 @@ func actualizeHpaTargetState(ctx hpaScopedContext) error { if ctx.hpa.Status.CurrentReplicas == 0 { //kube will not return current values if amount of replicas is 0, so we need to check every metric manually metricValues, err = requestMetricValuesFromSpec(ctx) + + if err != nil { + return fmt.Errorf("unable to determine if scaling from 0 is required: %s", err) + } } else { metricValues, err = extractMetricValuesFromCurrentMetrics(ctx.hpa) - } - if err != nil { - return err + if err != nil { + return fmt.Errorf("unable to determine if scaling to 0 is required: %s", err) + } } allAreZero := checkAllAreZero(metricValues) @@ -336,7 +346,7 @@ func actualizeHpaState(ctx context.Context, if hpa.ObjectMeta.CreationTimestamp.Time.Add(3 * time.Minute).After(time.Now()) { ctx.logger.Info(fmt.Sprintf("Not able to process newly-created HPA: %s", err)) } else { - ctx.logger.Error(err, "not able to process HPA: %s") + ctx.logger.Error(err, "not able to process HPA") } } }