diff --git a/.gitignore b/.gitignore index e549f085..e4a85bea 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ github_deploy_key *.json *.jwt -/go.work -/go.work.sum +go.work +go.work.sum diff --git a/edge-apis/pool.go b/edge-apis/pool.go index 0cc46478..15a2b927 100644 --- a/edge-apis/pool.go +++ b/edge-apis/pool.go @@ -18,6 +18,7 @@ package edge_apis import ( "github.com/go-openapi/runtime" + "github.com/michaelquigley/pfxlog" cmap "github.com/orcaman/concurrent-map/v2" "github.com/pkg/errors" "golang.org/x/exp/rand" @@ -61,6 +62,7 @@ type keyedClientTransport struct { } func (c *ClientTransportPoolRandom) setCurrent(key string, transport runtime.ClientTransport) { + pfxlog.Logger().WithField("key", key).Info("setting current controller to: " + key) c.current.Store(&keyedClientTransport{ ClientTransport: transport, key: key, @@ -78,58 +80,76 @@ func (c *ClientTransportPoolRandom) Remove(apiUrl *url.URL) { func (c *ClientTransportPoolRandom) Submit(operation *runtime.ClientOperation) (interface{}, error) { current := c.current.Load() - key := "" + currentKey := "" if current != nil { - key = current.key + currentKey = current.key result, err := (*current).Submit(operation) if err == nil || !isNetworkError(err) { + pfxlog.Logger().WithError(err).Debugf("determine a network error did not occur on (%T) returning", err) return result, err } + pfxlog.Logger().WithError(err).Debugf("encountered network error (%T) while submitting request", err) + if c.pool.Count() == 1 { + pfxlog.Logger().Debug("current transport not set and pool count is 1, returning error") + return result, err } } - if c.pool.Count() == 0 { - return nil, errors.New("no client transport available") - } - - return c.tryRandom(key, operation) + pfxlog.Logger().Debug("trying random transports from pool") + return c.tryRandom(currentKey, operation) } -func (c *ClientTransportPoolRandom) tryRandom(failed string, operation *runtime.ClientOperation) (interface{}, error) { +func (c *ClientTransportPoolRandom) tryRandom(skipKey string, operation *runtime.ClientOperation) (interface{}, error) { var curTransTpl *cmap.Tuple[string, runtime.ClientTransport] var lastResult any var lastErr error - var transportTpls []*cmap.Tuple[string, runtime.ClientTransport] + var transportsToTry []*cmap.Tuple[string, runtime.ClientTransport] for tpl := range c.pool.IterBuffered() { - if tpl.Key != failed { - transportTpls = append(transportTpls, &tpl) + if tpl.Key != skipKey { + transportsToTry = append(transportsToTry, &tpl) } } - for len(transportTpls) > 0 { - curTransTpl, transportTpls = selectAndRemoveRandom(transportTpls, nil) + pfxlog.Logger().Debugf("have %d transports to try", len(transportsToTry)) + + for len(transportsToTry) > 0 { + curTransTpl, transportsToTry = selectAndRemoveRandom(transportsToTry, nil) lastResult, lastErr = curTransTpl.Val.Submit(operation) if lastErr == nil || !isNetworkError(lastErr) { + pfxlog.Logger().WithField("key", curTransTpl.Key).Debug("transport success, setting new controller") c.setCurrent(curTransTpl.Key, curTransTpl.Val) return lastResult, lastErr } + + pfxlog.Logger().WithError(lastErr).Debugf("during tryRandom, encountered network error (%T) while submitting request, trying next", lastErr) } + pfxlog.Logger().Debug("after trying random transports from the pool, none succeeded, returning last error") + return lastResult, lastErr } var _ runtime.ClientTransport = (*ClientTransportPoolRandom)(nil) var _ ClientTransportPool = (*ClientTransportPoolRandom)(nil) +var opError = &net.OpError{} + func isNetworkError(err error) bool { - return errors.Is(err, &net.OpError{}) + pfxlog.Logger().WithError(err).Debugf("checking for network errror on type (%T) and its wrapped errors", err) + + if errors.As(err, &opError) { + pfxlog.Logger().Debug("detected net.OpError") + return true + } + + return false } func selectAndRemoveRandom[T any](slice []T, zero T) (selected T, modifiedSlice []T) {