Rate limit github clone calls (#142)

* feat: rate limit github clone calls

Add rate limiting to github clone calls, avoiding issues
with github "permission denied" and EOF errors when cloning too many repositories in parallel.

Fixes #139

* chore(config): initialize limiter with default buffer

Author: cbrand

README.md

@@ -41,6 +41,11 @@ func parseGitXargsConfig(c *cli.Context) (*config.GitXargsConfig, error) {
 	config.SecondsToSleepBetweenPRs = c.Int("seconds-between-prs")
 	config.PullRequestRetries = c.Int("max-pr-retries")
 	config.SecondsToSleepWhenRateLimited = c.Int("seconds-to-wait-when-rate-limited")
+	maxConcurrentClones := c.Int("max-concurrent-clones")
+	if maxConcurrentClones > 0 {
+		config.CloneJobsLimiter = make(chan struct{}, maxConcurrentClones)
+	}
+
 	config.NoSkipCI = c.Bool("no-skip-ci")
 	config.RetainLocalRepos = c.Bool("keep-cloned-repositories")
 	// By default, prepend "[skip ci]" to commit messages, unless the user passed --no-skip-ci

cmd/git-xargs.go

@@ -23,9 +23,10 @@ const (
 	DefaultPullRequestDescription        = "git-xargs programmatic pull request"
 	MaxPullRequestRetriesFlagName        = "max-pr-retries"
 	SecondsToWaitWhenRateLimitedFlagName = "seconds-to-wait-when-rate-limited"
+	MaxConcurrentClonesFlagName          = "max-concurrent-clones"
 	NoSkipCIFlagName                     = "no-skip-ci"
 	KeepClonedRepositoriesFlagName       = "keep-cloned-repositories"
-	DefaultMaxConcurrentRepos            = 0
+	DefaultMaxConcurrentClones           = 4
 	DefaultSecondsBetweenPRs             = 1
 	DefaultMaxPullRequestRetries         = 3
 	DefaultSecondsToWaitWhenRateLimited  = 60
@@ -106,6 +107,11 @@ var (
 		Usage: "The number of additional seconds to sleep before attempting to open a PR again, when rate limited by GitHub. Defaults to 60.",
 		Value: DefaultSecondsToWaitWhenRateLimited,
 	}
+	GenericMaxConcurrentClonesFlag = cli.IntFlag{
+		Name:  MaxConcurrentClonesFlagName,
+		Usage: "The maximum number of concurrent clones to run at once. Defaults to 4. If set to 0 no limit will be applied.",
+		Value: DefaultMaxConcurrentClones,
+	}
 	GenericNoSkipCIFlag = cli.BoolFlag{
 		Name:  NoSkipCIFlagName,
 		Usage: "By default, git-xargs prepends \"[skip ci]\" to its commit messages. Pass this flag to prevent \"[skip ci]\" from being prepending to commit messages.",

common/common.go

@@ -38,6 +38,7 @@ type GitXargsConfig struct {
 	SecondsToSleepBetweenPRs      int
 	PullRequestRetries            int
 	SecondsToSleepWhenRateLimited int
+	CloneJobsLimiter              chan struct{}
 	NoSkipCI                      bool
 	RetainLocalRepos              bool
 	Ticker                        *time.Ticker
@@ -70,6 +71,7 @@ func NewGitXargsConfig() *GitXargsConfig {
 		SecondsToSleepBetweenPRs:      common.DefaultSecondsBetweenPRs,
 		SecondsToSleepWhenRateLimited: common.DefaultSecondsToWaitWhenRateLimited,
 		PullRequestRetries:            common.DefaultMaxPullRequestRetries,
+		CloneJobsLimiter:              make(chan struct{}, common.DefaultMaxConcurrentClones),
 		NoSkipCI:                      false,
 		RetainLocalRepos:              false,
 	}

config/config.go

@@ -76,6 +76,7 @@ func setupApp() *cli.App {
 		common.GenericSecondsToWaitFlag,
 		common.GenericMaxPullRequestRetriesFlag,
 		common.GenericSecondsToWaitWhenRateLimitedFlag,
+		common.GenericMaxConcurrentClonesFlag,
 		common.GenericNoSkipCIFlag,
 		common.GenericKeepClonedRepositoriesFlag,
 	}

main.go

@@ -30,6 +30,7 @@ import (
 // git-xargs-<repo-name> appended to it to make it easier to find when you are looking for it while debugging
 func cloneLocalRepository(config *config.GitXargsConfig, repo *github.Repository) (string, *git.Repository, error) {
 	logger := logging.GetLogger("git-xargs")
+	config.CloneJobsLimiter <- struct{}{}
 
 	logger.WithFields(logrus.Fields{
 		"Repo": repo.GetName(),
@@ -58,6 +59,8 @@ func cloneLocalRepository(config *config.GitXargsConfig, repo *github.Repository
 		"Repo": repo.GetName(),
 	}).Debug(gitProgressBuffer)
 
+	<-config.CloneJobsLimiter
+
 	if err != nil {
 		logger.WithFields(logrus.Fields{
 			"Error": err,