Merge pull request #229 from transifex/retry_handle_additions

TX CLI pull job fails with a throttling message
transifex · Jun 21, 2024 · 3b0c3d9 · 3b0c3d9
2 parents fc80657 + 3ad1c52
commit 3b0c3d9
Show file tree

Hide file tree

Showing 4 changed files with 105 additions and 35 deletions.
diff --git a/internal/txlib/pull.go b/internal/txlib/pull.go
@@ -178,7 +178,17 @@ func (task *ResourcePullTask) Run(send func(string), abort func()) {
 	)
 
 	var err error
-	resource, err := txapi.GetResourceById(api, cfgResource.GetAPv3Id())
+	var resource *jsonapi.Resource
+	err = handleRetry(
+		func() error {
+			var err error
+			resource, err = txapi.GetResourceById(api, cfgResource.GetAPv3Id())
+			return err
+		},
+		"Getting info",
+		func(msg string) { sendMessage(msg, false) },
+	)
+
 	if err != nil {
 		sendMessage(err.Error(), true)
 		if !args.Skip {
@@ -210,11 +220,19 @@ func (task *ResourcePullTask) Run(send func(string), abort func()) {
 	sourceLanguage := project.Relationships["source_language"].DataSingular
 
 	var stats map[string]*jsonapi.Resource
-	if args.Source && !args.Translations {
-		stats, err = txapi.GetResourceStats(api, resource, sourceLanguage)
-	} else {
-		stats, err = txapi.GetResourceStats(api, resource, nil)
-	}
+	err = handleRetry(
+		func() error {
+			if args.Source && !args.Translations {
+				stats, err = txapi.GetResourceStats(api, resource, sourceLanguage)
+			} else {
+				stats, err = txapi.GetResourceStats(api, resource, nil)
+			}
+			return err
+		},
+		"Get language stats",
+		func(msg string) { sendMessage(msg, false) },
+	)
+
 	if err != nil {
 		sendMessage(err.Error(), true)
 		if !args.Skip {

diff --git a/internal/txlib/push.go b/internal/txlib/push.go
@@ -290,8 +290,17 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) {
 		}
 		send(message)
 	}
-	sendMessage("Getting info", false)
-	resource, err := txapi.GetResourceById(api, cfgResource.GetAPv3Id())
+
+	var resource *jsonapi.Resource
+	err := handleRetry(
+		func() error {
+			var err error
+			resource, err = txapi.GetResourceById(api, cfgResource.GetAPv3Id())
+			return err
+		},
+		"Getting info",
+		func(msg string) { sendMessage(msg, false) },
+	)
 	if err != nil {
 		sendMessage(fmt.Sprintf("Error while fetching resource: %s", err), true)
 		if !args.Skip {
@@ -341,7 +350,18 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) {
 				cfgResource.ProjectSlug,
 				baseResourceSlug,
 			)
-			baseResource, err := txapi.GetResourceById(api, baseResourceId)
+
+			var baseResource *jsonapi.Resource
+			err = handleRetry(
+				func() error {
+					var err error
+					baseResource, err = txapi.GetResourceById(api, baseResourceId)
+					return err
+				},
+				"Getting info",
+				func(msg string) { sendMessage(msg, false) },
+			)
+
 			if err != nil {
 				sendMessage(fmt.Sprintf("Error while fetching base resource: %s", err), true)
 				if !args.Skip {
@@ -364,18 +384,27 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) {
 			}
 		}
 
-		resource, err = txapi.CreateResource(
-			api,
-			fmt.Sprintf(
-				"o:%s:p:%s",
-				cfgResource.OrganizationSlug,
-				cfgResource.ProjectSlug,
-			),
-			resourceName,
-			cfgResource.ResourceSlug,
-			cfgResource.Type,
-			baseResourceId,
+		err = handleRetry(
+			func() error {
+				var err error
+				resource, err = txapi.CreateResource(
+					api,
+					fmt.Sprintf(
+						"o:%s:p:%s",
+						cfgResource.OrganizationSlug,
+						cfgResource.ProjectSlug,
+					),
+					resourceName,
+					cfgResource.ResourceSlug,
+					cfgResource.Type,
+					baseResourceId,
+				)
+				return err
+			},
+			"Create resource",
+			func(msg string) { sendMessage(msg, false) },
 		)
+
 		if err != nil {
 			sendMessage(fmt.Sprintf("Error while creating resource, %s", err), true)
 			if !args.Skip {
@@ -431,11 +460,19 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) {
 	}
 	sourceLanguage := sourceLanguageRelationship.DataSingular
 	var remoteStats map[string]*jsonapi.Resource
-	if args.Translation {
-		remoteStats, err = txapi.GetResourceStats(api, resource, nil)
-	} else {
-		remoteStats, err = txapi.GetResourceStats(api, resource, sourceLanguage)
-	}
+	err = handleRetry(
+		func() error {
+			var err error
+			if args.Translation {
+				remoteStats, err = txapi.GetResourceStats(api, resource, nil)
+			} else {
+				remoteStats, err = txapi.GetResourceStats(api, resource, sourceLanguage)
+			}
+			return err
+		},
+		"Getting resource stats",
+		func(msg string) { sendMessage(msg, false) },
+	)
 	if err != nil {
 		sendMessage(fmt.Sprintf("Error while fetching stats, %s", err), true)
 		if !args.Skip {
@@ -496,7 +533,16 @@ func (task *ResourcePushTask) Run(send func(string), abort func()) {
 			return
 		}
 
-		allLanguages, err := txapi.GetLanguages(api)
+		var allLanguages map[string]*jsonapi.Resource
+		err = handleRetry(
+			func() error {
+				var err error
+				allLanguages, err = txapi.GetLanguages(api)
+				return err
+			},
+			"Getting languages",
+			func(msg string) { sendMessage(msg, false) },
+		)
 		if err != nil {
 			sendMessage(err.Error(), true)
 			abort()

diff --git a/internal/txlib/utils.go b/internal/txlib/utils.go
@@ -171,17 +171,15 @@ func handleRetry(do func() error, initialMsg string, send func(string)) error {
 				retryAfter := e.RetryAfter
 				if isatty.IsTerminal(os.Stdout.Fd()) {
 					for retryAfter > 0 {
-						send(fmt.Sprintf(
-							"will retry after %d seconds",
-							retryAfter,
+						send(fmt.Sprint(
+							err,
 						))
 						time.Sleep(time.Second)
 						retryAfter -= 1
 					}
 				} else {
-					send(fmt.Sprintf(
-						"will retry after %d seconds",
-						retryAfter,
+					send(fmt.Sprint(
+						err,
 					))
 					time.Sleep(time.Duration(retryAfter) * time.Second)
 				}

diff --git a/pkg/jsonapi/errors.go b/pkg/jsonapi/errors.go
@@ -78,11 +78,14 @@ func (m *RedirectError) Error() string {
 }
 
 type RetryError struct {
+	StatusCode int
 	RetryAfter int
 }
 
 func (err RetryError) Error() string {
-	return fmt.Sprintf("throttled; retry after %d", err.RetryAfter)
+	return fmt.Sprintf(
+		"Response error code %d, retry after %d", err.StatusCode, err.RetryAfter,
+	)
 }
 
 func parseRetryResponse(response *http.Response) *RetryError {
@@ -92,9 +95,14 @@ func parseRetryResponse(response *http.Response) *RetryError {
 		response.StatusCode != 504 {
 		return nil
 	}
+	if response.StatusCode == 502 ||
+		response.StatusCode == 503 ||
+		response.StatusCode == 504 {
+		return &RetryError{response.StatusCode, 10}
+	}
 	retryAfter, err := strconv.Atoi(response.Header.Get("Retry-After"))
 	if err != nil {
-		return &RetryError{1}
+		return &RetryError{response.StatusCode, 1}
 	}
-	return &RetryError{retryAfter}
+	return &RetryError{response.StatusCode, retryAfter}
 }