Prevent metal-hammer blocking long when Loki is unavailable (#132)

cmd/remote-logging.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"time"
 
 	"github.com/grafana/loki-client-go/loki"
 	"github.com/metal-stack/pixie/api"
@@ -41,23 +42,32 @@ func AddRemoteHandler(spec *Specification, handler slog.Handler) (slog.Handler,
 	if err != nil {
 		return nil, fmt.Errorf("unable to create loki default config %w", err)
 	}
+
 	// config.EncodeJson = true
 	config.Client = httpClient
+	// we need to set small timeout here otherwise loki can block hammer execution
+	config.BackoffConfig.MinBackoff = 100 * time.Millisecond
+	config.BackoffConfig.MaxBackoff = 1 * time.Second
+	config.BackoffConfig.MaxRetries = 1
+
 	client, err := loki.New(config)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create loki client %w", err)
 	}
 
-	lokiHandler := slogloki.Option{
-		Level:  slog.LevelDebug,
-		Client: client}.NewLokiHandler().WithAttrs(
-		[]slog.Attr{
-			{Key: "component", Value: slog.StringValue("metal-hammer")},
-			{Key: "machineID", Value: slog.StringValue(spec.MachineUUID)},
-		},
+	var (
+		lokiHandler = slogloki.Option{
+			Level:  slog.LevelDebug,
+			Client: client}.NewLokiHandler().WithAttrs(
+			[]slog.Attr{
+				{Key: "component", Value: slog.StringValue("metal-hammer")},
+				{Key: "machineID", Value: slog.StringValue(spec.MachineUUID)},
+			},
+		)
+		mdw             = slogmulti.NewHandleInlineMiddleware(jsonFormattingMiddleware)
+		combinedHandler = slogmulti.Fanout(slogmulti.Pipe(mdw).Handler(lokiHandler), handler)
 	)
-	mdw := slogmulti.NewHandleInlineMiddleware(jsonFormattingMiddleware)
-	combinedHandler := slogmulti.Fanout(slogmulti.Pipe(mdw).Handler(lokiHandler), handler)
+
 	return combinedHandler, nil
 }
 

cmd/root.go

@@ -179,6 +179,11 @@ func (h *hammer) installImage(eventEmitter *event.EventEmitter, bootService v1.B
 	}
 
 	h.log.Info("installation", "took", time.Since(installationStart))
+
+	// this can be useful for metal-images os debugging
+	// h.log.Info("waiting 10 sec to enable os debugging")
+	// time.Sleep(10 * time.Second)
+
 	eventEmitter.Emit(event.ProvisioningEventBootingNewKernel, "booting into distro kernel")
 	return kernel.RunKexec(info)
 }

go.mod

@@ -11,23 +11,23 @@ require (
 	github.com/jaypipes/ghw v0.12.0
 	github.com/metal-stack/go-hal v0.5.3
 	github.com/metal-stack/go-lldpd v0.4.7
-	github.com/metal-stack/metal-api v0.31.0
-	github.com/metal-stack/metal-go v0.31.0
+	github.com/metal-stack/metal-api v0.32.2
+	github.com/metal-stack/metal-go v0.32.2
 	github.com/metal-stack/pixie v0.3.2
 	github.com/metal-stack/v v1.0.3
 	// archiver must stay in version v2.1.0, see replace below
 	github.com/mholt/archiver v3.1.1+incompatible
 	github.com/moby/sys/mountinfo v0.7.1
 	github.com/pierrec/lz4/v4 v4.1.21
-	github.com/prometheus/common v0.53.0
-	github.com/samber/slog-loki/v3 v3.3.0
-	github.com/samber/slog-multi v1.0.3
+	github.com/prometheus/common v0.55.0
+	github.com/samber/slog-loki/v3 v3.5.0
+	github.com/samber/slog-multi v1.2.0
 	github.com/u-root/u-root v0.14.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.20.0
-	google.golang.org/grpc v1.64.0
-	google.golang.org/protobuf v1.34.1
+	golang.org/x/sys v0.22.0
+	google.golang.org/grpc v1.65.0
+	google.golang.org/protobuf v1.34.2
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -45,7 +45,7 @@ require (
 	github.com/avast/retry-go/v4 v4.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/coreos/go-oidc/v3 v3.10.0 // indirect
+	github.com/coreos/go-oidc/v3 v3.11.0 // indirect
 	github.com/creack/pty v1.1.21 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
@@ -54,7 +54,7 @@ require (
 	github.com/frankban/quicktest v1.14.6 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/gliderlabs/ssh v0.3.7 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.3 // indirect
 	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
@@ -77,30 +77,29 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/jaypipes/pcidb v1.0.0 // indirect
+	github.com/jaypipes/pcidb v1.0.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.2 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/httprc v1.0.5 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
-	github.com/lestrrat-go/jwx/v2 v2.0.21 // indirect
+	github.com/lestrrat-go/jwx/v2 v2.1.0 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect
 	github.com/mdlayher/lldp v0.0.0-20150915211757-afd9f83164c5 // indirect
-	github.com/metal-stack/metal-lib v0.16.2 // indirect
-	github.com/metal-stack/security v0.8.0 // indirect
+	github.com/metal-stack/metal-lib v0.17.1 // indirect
+	github.com/metal-stack/security v0.8.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
@@ -109,29 +108,29 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/procfs v0.15.0 // indirect
-	github.com/prometheus/prometheus v0.52.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/prometheus/prometheus v0.53.1 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
-	github.com/samber/lo v1.39.0 // indirect
-	github.com/samber/slog-common v0.16.0 // indirect
+	github.com/samber/lo v1.44.0 // indirect
+	github.com/samber/slog-common v0.17.1 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
-	github.com/sethvargo/go-password v0.3.0 // indirect
+	github.com/sethvargo/go-password v0.3.1 // indirect
 	github.com/stmcginnis/gofish v0.18.0 // indirect
 	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
 	github.com/ulikunitz/xz v0.5.12 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 	github.com/vmware/goipmi v0.0.0-20181114221114-2333cd82d702 // indirect
-	go.mongodb.org/mongo-driver v1.15.0 // indirect
-	go.opentelemetry.io/otel v1.27.0 // indirect
-	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+	go.mongodb.org/mongo-driver v1.16.0 // indirect
+	go.opentelemetry.io/otel v1.28.0 // indirect
+	go.opentelemetry.io/otel/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.23.0 // indirect
-	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/oauth2 v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect
+	golang.org/x/crypto v0.25.0 // indirect
+	golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
+	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240709173604-40e1e62336c5 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	howett.net/plist v1.0.1 // indirect
 )