Merge pull request #49 from gurkanguray/feat/17-retry-mechanism

Crossing fingers and landing it. -- worst case we can fix it later.

.gitignore

@@ -53,6 +53,8 @@ go.work.sum
 .idea
 .code
 .history
+*.DS_Store
+*.iml
 
 # App-specific:
 .spike-token

app/nexus/internal/env/backend.go

@@ -40,7 +40,6 @@ func BackendStoreType() StoreType {
 	switch strings.ToLower(st) {
 	case string(S3):
 		panic("SPIKE_NEXUS_BACKEND_STORE=s3 is not implemented yet")
-		return S3
 	case string(Sqlite):
 		return Sqlite
 	case string(Memory):

app/nexus/internal/state/persist/credentials.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spiffe/spike/app/nexus/internal/env"
 	"github.com/spiffe/spike/app/nexus/internal/state/entity/data"
 	"github.com/spiffe/spike/internal/log"
+	"github.com/spiffe/spike/pkg/retry"
 )
 
 // ReadAdminRecoveryMetadata retrieves cached admin recovery metadata from
@@ -27,22 +28,27 @@ func ReadAdminRecoveryMetadata() *data.RecoveryMetadata {
 		return nil
 	}
 
+	retrier := retry.NewExponentialRetrier()
+	typedRetrier := retry.NewTypedRetrier[data.RecoveryMetadata](retrier)
+
 	ctx, cancel := context.WithTimeout(
 		context.Background(), env.DatabaseOperationTimeout(),
 	)
 	defer cancel()
 
-	cachedMetadata, err := be.LoadAdminRecoveryMetadata(ctx)
+	metadata, err := typedRetrier.RetryWithBackoff(ctx, func() (data.RecoveryMetadata, error) {
+		return be.LoadAdminRecoveryMetadata(ctx)
+	})
+
 	if err != nil {
-		// Log error but continue - memory is source of truth
 		log.Log().Warn("readAdminRecoveryMetadata",
-			"msg", "Failed to load admin recovery metadata from cache",
+			"msg", "Failed to load admin recovery metadata after retries",
 			"err", err.Error(),
 		)
 		return nil
 	}
 
-	return &cachedMetadata
+	return &metadata
 }
 
 // AsyncPersistAdminRecoveryMetadata asynchronously stores admin recovery
@@ -61,16 +67,20 @@ func AsyncPersistAdminRecoveryMetadata(credentials data.RecoveryMetadata) {
 			return // No cache available
 		}
 
+		retrier := retry.NewExponentialRetrier()
+
 		ctx, cancel := context.WithTimeout(
-			context.Background(),
-			env.DatabaseOperationTimeout(),
+			context.Background(), env.DatabaseOperationTimeout(),
 		)
 		defer cancel()
 
-		if err := be.StoreAdminRecoveryMetadata(ctx, credentials); err != nil {
-			// Log error but continue - memory is source of truth
+		err := retrier.RetryWithBackoff(ctx, func() error {
+			return be.StoreAdminRecoveryMetadata(ctx, credentials)
+		})
+
+		if err != nil {
 			log.Log().Warn("asyncPersistAdminRecoveryMetadata",
-				"msg", "Failed to cache admin recovery metadata",
+				"msg", "Failed to cache admin recovery metadata after retries",
 				"err", err.Error(),
 			)
 		}

app/nexus/internal/state/persist/secret.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/spiffe/spike/app/nexus/internal/env"
 	"github.com/spiffe/spike/internal/log"
+	"github.com/spiffe/spike/pkg/retry"
 	"github.com/spiffe/spike/pkg/store"
 )
 
@@ -37,15 +38,21 @@ func ReadSecret(path string, version int) *store.Secret {
 		return nil
 	}
 
+	retrier := retry.NewExponentialRetrier()
+	typedRetrier := retry.NewTypedRetrier[*store.Secret](retrier)
+
 	ctx, cancel := context.WithTimeout(
 		context.Background(), env.DatabaseOperationTimeout(),
 	)
 	defer cancel()
 
-	cachedSecret, err := be.LoadSecret(ctx, path)
+	cachedSecret, err := typedRetrier.RetryWithBackoff(ctx, func() (*store.Secret, error) {
+		return be.LoadSecret(ctx, path)
+	})
+
 	if err != nil {
 		log.Log().Warn("readSecret",
-			"msg", "Failed to load secret from cache",
+			"msg", "Failed to load secret from cache after retries",
 			"path", path,
 			"err", err.Error(),
 		)

app/nexus/internal/state/persist/token.go

@@ -12,6 +12,7 @@ import (
 	"github.com/spiffe/spike/app/nexus/internal/state/backend/memory"
 	"github.com/spiffe/spike/app/nexus/internal/state/backend/sqlite"
 	"github.com/spiffe/spike/internal/log"
+	"github.com/spiffe/spike/pkg/retry"
 )
 
 var (
@@ -38,12 +39,18 @@ func ReadAdminSigningToken() string {
 		return ""
 	}
 
+	retrier := retry.NewExponentialRetrier()
+	typedRetrier := retry.NewTypedRetrier[string](retrier)
+
 	ctx, cancel := context.WithTimeout(
 		context.Background(), env.DatabaseOperationTimeout(),
 	)
 	defer cancel()
 
-	cachedToken, err := be.LoadAdminSigningToken(ctx)
+	cachedToken, err := typedRetrier.RetryWithBackoff(ctx, func() (string, error) {
+		return be.LoadAdminSigningToken(ctx)
+	})
+
 	if err != nil {
 		// Log error but continue - memory is source of truth
 		log.Log().Warn("readAdminSigningToken",
@@ -74,16 +81,21 @@ func AsyncPersistAdminToken(token string) {
 			return // No cache available
 		}
 
+		retrier := retry.NewExponentialRetrier()
+
 		ctx, cancel := context.WithTimeout(
 			context.Background(),
 			env.DatabaseOperationTimeout(),
 		)
 		defer cancel()
 
-		if err := be.StoreAdminToken(ctx, token); err != nil {
-			// Log error but continue - memory is source of truth
+		err := retrier.RetryWithBackoff(ctx, func() error {
+			return be.StoreAdminToken(ctx, token)
+		})
+
+		if err != nil {
 			log.Log().Warn("asyncPersistAdminToken",
-				"msg", "Failed to cache admin token",
+				"msg", "Failed to cache admin token after retries",
 				"err", err.Error(),
 			)
 		}

app/spike/internal/cmd/system/init.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spiffe/spike/app/spike/internal/net/auth"
 	"github.com/spiffe/spike/internal/config"
 	"github.com/spiffe/spike/internal/entity/data"
+	"github.com/spiffe/spike/pkg/retry"
 )
 
 // NewSystemInitCommand creates and returns a new cobra.Command for initializing
@@ -45,7 +46,13 @@ func NewSystemInitCommand(source *workloadapi.X509Source) *cobra.Command {
 		Use:   "init",
 		Short: "Initialize spike configuration",
 		Run: func(cmd *cobra.Command, args []string) {
-			state, err := auth.CheckInitState(source)
+			retrier := retry.NewExponentialRetrier()
+			typedRetrier := retry.NewTypedRetrier[data.InitState](retrier)
+
+			ctx := cmd.Context()
+			state, err := typedRetrier.RetryWithBackoff(ctx, func() (data.InitState, error) {
+				return auth.CheckInitState(source)
+			})
 
 			if err != nil {
 				fmt.Println("Failed to check initialization state:")
@@ -59,7 +66,10 @@ func NewSystemInitCommand(source *workloadapi.X509Source) *cobra.Command {
 				return
 			}
 
-			err = auth.Init(source)
+			err = retrier.RetryWithBackoff(ctx, func() error {
+				return auth.Init(source)
+			})
+
 			if err != nil {
 				fmt.Println("Failed to save admin token:")
 				fmt.Println(err.Error())

go.mod

@@ -3,19 +3,24 @@ module github.com/spiffe/spike
 go 1.23.2
 
 require (
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/go-jose/go-jose/v4 v4.0.4
 	github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
 	github.com/google/uuid v1.6.0
 	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/spf13/cobra v1.8.1
 	github.com/spiffe/go-spiffe/v2 v2.4.0
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/crypto v0.26.0
 )
 
 require (
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
@@ -24,4 +29,5 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
 	google.golang.org/grpc v1.67.1 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -2,10 +2,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -32,6 +35,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
 github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -98,6 +105,8 @@ google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFN
 google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
 google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/retry/mock/mock.go

@@ -0,0 +1,25 @@
+//    \\ SPIKE: Secure your secrets with SPIFFE.
+//  \\\\\ Copyright 2024-present SPIKE contributors.
+// \\\\\\\ SPDX-License-Identifier: Apache-2.0
+
+package mock
+
+import (
+	"context"
+)
+
+// MockRetrier implements Retrier for testing
+type MockRetrier struct {
+	RetryFunc func(context.Context, func() error) error
+}
+
+// RetryWithBackoff implements the Retrier interface
+func (m *MockRetrier) RetryWithBackoff(
+	ctx context.Context,
+	operation func() error,
+) error {
+	if m.RetryFunc != nil {
+		return m.RetryFunc(ctx, operation)
+	}
+	return nil
+}