Add exponential backoff for requests that receive PV drivers missing …

…error

client/client.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v3"
 	gorillawebsocket "github.com/gorilla/websocket"
 	"github.com/sourcegraph/jsonrpc2"
 	"github.com/sourcegraph/jsonrpc2/websocket"
@@ -228,33 +229,56 @@ func NewClient(config Config) (XOClient, error) {
 	}, nil
 }
 
-func (c *Client) Call(method string, params, result interface{}, opt ...jsonrpc2.CallOption) error {
-	err := c.rpc.Call(context.Background(), method, params, result, opt...)
-	var callRes interface{}
-	t := reflect.TypeOf(result)
-	if t == nil || t.Kind() != reflect.Ptr {
-		callRes = result
-	} else {
-		callRes = reflect.ValueOf(result).Elem()
-	}
-	log.Printf("[TRACE] Made rpc call `%s` with params: %v and received %+v: result with error: %v\n", method, params, callRes, err)
+func IsRetryableError(err jsonrpc2.Error) bool {
+	// Error code 11 corresponds to an error condition where a VM is missing PV drivers.
+	// https://github.com/vatesfr/xen-orchestra/blob/a3a2fda157fa30af4b93d34c99bac550f7c82bbc/packages/xo-common/api-errors.js#L95
 
-	if err != nil {
-		rpcErr, ok := err.(*jsonrpc2.Error)
+	// During the boot process, there is a race condition where the PV drivers aren't available yet and
+	// making XO api calls during this time can return a VM_MISSING_PV_DRIVERS error. These errors can
+	// be treated as retryable since we want to wait until the VM has finished booting and its PV driver
+	// is initialized.
+	if err.Code == 11 {
+		return true
+	}
+	return false
+}
 
-		if !ok {
-			return err
+func (c *Client) Call(method string, params, result interface{}) error {
+	operation := func() error {
+		err := c.rpc.Call(context.Background(), method, params, result)
+		var callRes interface{}
+		t := reflect.TypeOf(result)
+		if t == nil || t.Kind() != reflect.Ptr {
+			callRes = result
+		} else {
+			callRes = reflect.ValueOf(result).Elem()
 		}
+		log.Printf("[TRACE] Made rpc call `%s` with params: %v and received %+v: result with error: %v\n", method, params, callRes, err)
+
+		if err != nil {
+			rpcErr, ok := err.(*jsonrpc2.Error)
 
-		data := rpcErr.Data
+			if !ok {
+				return backoff.Permanent(err)
+			}
 
-		if data == nil {
-			return err
-		}
+			if IsRetryableError(*rpcErr) {
+				return err
+			}
+
+			data := rpcErr.Data
 
-		return errors.New(fmt.Sprintf("%s: %s", err, *data))
+			if data == nil {
+				return backoff.Permanent(err)
+			}
+
+			return backoff.Permanent(errors.New(fmt.Sprintf("%s: %s", err, *data)))
+		}
+		return nil
 	}
-	return nil
+
+	bo := backoff.NewExponentialBackOff()
+	return backoff.Retry(operation, bo)
 }
 
 type RefreshComparison interface {

go.mod

@@ -3,6 +3,7 @@ module github.com/ddelnano/terraform-provider-xenorchestra
 go 1.16
 
 require (
+	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/ddelnano/terraform-provider-xenorchestra/client v0.0.0-00010101000000-000000000000
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.4.3
 )

go.sum

@@ -56,6 +56,8 @@ github.com/aws/aws-sdk-go v1.25.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
 github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M=
+github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=