diff --git a/Makefile b/Makefile index 4405fc3..d571aa6 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ fmt: goimports -w . && gofumpt -l -w . lint: - golangci-lint run --disable lll --disable interfacer --disable gochecknoglobals --disable gochecknoinits --enable wsl --enable revive --enable gosec --enable exhaustruct --enable unused --enable gocritic --enable gofmt --enable goimports --enable misspell --enable unparam --enable goconst --enable wrapcheck + golangci-lint run --disable lll --disable interfacer --disable gochecknoglobals --disable gochecknoinits --enable wsl --enable revive --enable gosec --enable unused --enable gocritic --enable gofmt --enable goimports --enable misspell --enable unparam --enable goconst --enable wrapcheck ci: lint test BUILD_TAG := $(shell git describe --tags 2>/dev/null) diff --git a/bitbucket.go b/bitbucket.go new file mode 100644 index 0000000..6092110 --- /dev/null +++ b/bitbucket.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + + "gitlab.com/tozd/go/errors" + + "github.com/jonhadfield/githosts-utils" +) + +func Bitbucket(backupDir string) *ProviderBackupResults { + logger.Println("backing up BitBucket repos") + + bitbucketHost, err := githosts.NewBitBucketHost(githosts.NewBitBucketHostInput{ + Caller: appName, + APIURL: os.Getenv(envBitBucketAPIURL), + DiffRemoteMethod: os.Getenv(envBitBucketCompare), + BackupDir: backupDir, + User: os.Getenv(envBitBucketUser), + Key: os.Getenv(envBitBucketKey), + Secret: os.Getenv(envBitBucketSecret), + BackupsToRetain: getBackupsToRetain(envBitBucketBackups), + LogLevel: getLogLevel(), + }) + if err != nil { + return &ProviderBackupResults{ + Provider: providerNameBitBucket, + Results: githosts.ProviderBackupResult{ + BackupResults: []githosts.RepoBackupResults{}, + Error: errors.Wrap(err, "failed to create BitBucket host"), + }, + } + } + + return &ProviderBackupResults{ + Provider: providerNameBitBucket, + Results: bitbucketHost.Backup(), + } +} diff --git a/gitea.go b/gitea.go new file mode 100644 index 0000000..6173dc6 --- /dev/null +++ b/gitea.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" + + "github.com/jonhadfield/githosts-utils" + "gitlab.com/tozd/go/errors" +) + +func Gitea(backupDir string) *ProviderBackupResults { + logger.Println("backing up Gitea repos") + + giteaHost, err := githosts.NewGiteaHost(githosts.NewGiteaHostInput{ + Caller: appName, + APIURL: os.Getenv(envGiteaAPIURL), + DiffRemoteMethod: os.Getenv(envGiteaCompare), + BackupDir: backupDir, + Token: os.Getenv(envGiteaToken), + Orgs: getOrgsListFromEnvVar(envGiteaOrgs), + BackupsToRetain: getBackupsToRetain(envGiteaBackups), + LogLevel: getLogLevel(), + }) + if err != nil { + return &ProviderBackupResults{ + Provider: providerNameGitea, + Results: githosts.ProviderBackupResult{ + BackupResults: []githosts.RepoBackupResults{}, + Error: errors.Wrap(err, "failed to create Gitea host"), + }, + } + } + + return &ProviderBackupResults{ + Provider: providerNameGitea, + Results: giteaHost.Backup(), + } +} diff --git a/github.go b/github.go new file mode 100644 index 0000000..d83f866 --- /dev/null +++ b/github.go @@ -0,0 +1,39 @@ +package main + +import ( + "os" + + "github.com/jonhadfield/githosts-utils" + + "gitlab.com/tozd/go/errors" +) + +func GitHub(backupDir string) *ProviderBackupResults { + logger.Println("backing up GitHub repos") + + githubHost, err := githosts.NewGitHubHost(githosts.NewGitHubHostInput{ + Caller: appName, + APIURL: os.Getenv(envGitHubAPIURL), + DiffRemoteMethod: os.Getenv(envGitHubCompare), + BackupDir: backupDir, + Token: os.Getenv(envGitHubToken), + Orgs: getOrgsListFromEnvVar(envGitHubOrgs), + BackupsToRetain: getBackupsToRetain(envGitHubBackups), + SkipUserRepos: envTrue(envGitHubSkipUserRepos), + LogLevel: getLogLevel(), + }) + if err != nil { + return &ProviderBackupResults{ + Provider: providerNameGitHub, + Results: githosts.ProviderBackupResult{ + BackupResults: []githosts.RepoBackupResults{}, + Error: errors.Wrap(err, "failed to create GitHub host"), + }, + } + } + + return &ProviderBackupResults{ + Provider: providerNameGitHub, + Results: githubHost.Backup(), + } +} diff --git a/gitlab.go b/gitlab.go new file mode 100644 index 0000000..4711e3f --- /dev/null +++ b/gitlab.go @@ -0,0 +1,40 @@ +package main + +import ( + "os" + + "gitlab.com/tozd/go/errors" + + "github.com/jonhadfield/githosts-utils" +) + +func Gitlab(backupDir string) *ProviderBackupResults { + logger.Println("backing up GitLab repos") + + var gitlabHost *githosts.GitLabHost + + gitlabHost, err := githosts.NewGitLabHost(githosts.NewGitLabHostInput{ + Caller: appName, + APIURL: os.Getenv(envGitLabAPIURL), + DiffRemoteMethod: os.Getenv(envGitLabCompare), + BackupDir: backupDir, + Token: os.Getenv(envGitLabToken), + BackupsToRetain: getBackupsToRetain(envGitLabBackups), + ProjectMinAccessLevel: getProjectMinimumAccessLevel(), + LogLevel: getLogLevel(), + }) + if err != nil { + return &ProviderBackupResults{ + Provider: providerNameGitLab, + Results: githosts.ProviderBackupResult{ + BackupResults: []githosts.RepoBackupResults{}, + Error: errors.Wrap(err, "failed to create GitLab host"), + }, + } + } + + return &ProviderBackupResults{ + Provider: providerNameGitLab, + Results: gitlabHost.Backup(), + } +} diff --git a/go.mod b/go.mod index 21d7a51..3f88bf9 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,10 @@ toolchain go1.21.0 require ( github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 github.com/hashicorp/go-retryablehttp v0.7.5 - github.com/jonhadfield/githosts-utils v0.0.0-20240214205906-bdeb9684d45c + github.com/jonhadfield/githosts-utils v0.0.0-20240227215907-fdbfc9a27143 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.8.4 + gitlab.com/tozd/go/errors v0.8.1 gopkg.in/h2non/gock.v1 v1.1.2 ) @@ -20,7 +21,6 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - gitlab.com/tozd/go/errors v0.8.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 094aec3..5576ede 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= -github.com/jonhadfield/githosts-utils v0.0.0-20240214205906-bdeb9684d45c h1:JVRdnCB5SsFyc4D8zhC5FVBJDvayIPL0MZEbau3TL+g= -github.com/jonhadfield/githosts-utils v0.0.0-20240214205906-bdeb9684d45c/go.mod h1:PFx5umZGByGGkaKyaw5E9az/IL8KiVjrDy9kymZQ6Oc= +github.com/jonhadfield/githosts-utils v0.0.0-20240227215907-fdbfc9a27143 h1:A8Kjq6LRh5HQiAGxkzQ99Nd1M4EyHS8eu3m3wurvpMg= +github.com/jonhadfield/githosts-utils v0.0.0-20240227215907-fdbfc9a27143/go.mod h1:2hre3/B2QsIOf6S69L3NCq1kkLu/7+TGFuIwNxDYIH4= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= diff --git a/main.go b/main.go index 83904c9..936754b 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "os" + "path/filepath" "runtime" "slices" "strconv" @@ -23,17 +24,19 @@ const ( workingDIRMode = 0o755 defaultBackupsToRetain = 2 defaultGitLabMinimumProjectAccessLevel = 20 + defaultEarlyErrorBackOffSeconds = 5 pathSep = string(os.PathSeparator) // env vars - envSobaLogLevel = "SOBA_LOG" - envSobaWebHookURL = "SOBA_WEBHOOK_URL" - envSobaWebHookFormat = "SOBA_WEBHOOK_FORMAT" - envGitBackupInterval = "GIT_BACKUP_INTERVAL" - envGitBackupDir = "GIT_BACKUP_DIR" - envGitHubAPIURL = "GITHUB_APIURL" - envGitHubBackups = "GITHUB_BACKUPS" + envSobaLogLevel = "SOBA_LOG" + envSobaWebHookURL = "SOBA_WEBHOOK_URL" + envSobaWebHookFormat = "SOBA_WEBHOOK_FORMAT" + envSobaEarlyErrorBackOff = "SOBA_EARLY_ERROR_BACKOFF" + envGitBackupInterval = "GIT_BACKUP_INTERVAL" + envGitBackupDir = "GIT_BACKUP_DIR" + envGitHubAPIURL = "GITHUB_APIURL" + envGitHubBackups = "GITHUB_BACKUPS" // nolint:gosec envGitHubToken = "GITHUB_TOKEN" envGitHubOrgs = "GITHUB_ORGS" @@ -133,18 +136,16 @@ func getBackupInterval() int { func getLogLevel() int { sobaLogLevelEnv := os.Getenv(envSobaLogLevel) - var sobaLogLevel int - - var intervalConversionErr error - if sobaLogLevelEnv != "" { - sobaLogLevel, intervalConversionErr = strconv.Atoi(sobaLogLevelEnv) - if intervalConversionErr != nil { + sobaLogLevel, err := strconv.Atoi(sobaLogLevelEnv) + if err != nil { logger.Fatalf("%s must be a number.", envSobaLogLevel) } + + return sobaLogLevel } - return sobaLogLevel + return 0 } func checkProviderFactory(provider string) func() { @@ -302,11 +303,7 @@ func displayStartupConfig() { func run() error { displayStartupConfig() - var backupDIR string - - var backupDIRKeyExists bool - - backupDIR, backupDIRKeyExists = os.LookupEnv(envGitBackupDir) + backupDIR, backupDIRKeyExists := os.LookupEnv(envGitBackupDir) if !backupDIRKeyExists || backupDIR == "" { return fmt.Errorf("environment variable %s must be set", envGitBackupDir) } @@ -317,7 +314,7 @@ func run() error { } } - backupDIR = stripTrailingLineBreak(backupDIR) + backupDIR = strings.TrimSuffix(backupDIR, "\n") _, err := os.Stat(backupDIR) if os.IsNotExist(err) { @@ -328,11 +325,7 @@ func run() error { logger.Fatal("no providers defined") } - if len(backupDIR) > 1 && strings.HasSuffix(backupDIR, "/") { - backupDIR = backupDIR[:len(backupDIR)-1] - } - - workingDIR := backupDIR + pathSep + workingDIRName + workingDIR := filepath.Join(backupDIR, workingDIRName) logger.Println("creating working directory:", workingDIR) createWorkingDIRErr := os.MkdirAll(workingDIR, workingDIRMode) @@ -409,112 +402,19 @@ func execProviderBackups() { var providerBackupResults []ProviderBackupResults if os.Getenv(envBitBucketUser) != "" { - logger.Println("backing up BitBucket repos") - - var bitbucketHost *githosts.BitbucketHost - - bitbucketHost, err = githosts.NewBitBucketHost(githosts.NewBitBucketHostInput{ - Caller: appName, - APIURL: os.Getenv(envBitBucketAPIURL), - DiffRemoteMethod: os.Getenv(envBitBucketCompare), - BackupDir: backupDir, - User: os.Getenv(envBitBucketUser), - Key: os.Getenv(envBitBucketKey), - Secret: os.Getenv(envBitBucketSecret), - BackupsToRetain: getBackupsToRetain(envBitBucketBackups), - LogLevel: getLogLevel(), - }) - if err != nil { - logger.Fatal(err) - } - - bitBucketResults := bitbucketHost.Backup() - - providerBackupResults = append(providerBackupResults, ProviderBackupResults{ - Provider: providerNameBitBucket, - Results: bitBucketResults, - }) + providerBackupResults = append(providerBackupResults, *Bitbucket(backupDir)) } if os.Getenv(envGiteaToken) != "" { - logger.Println("backing up Gitea repos") - - var giteaHost *githosts.GiteaHost - - giteaHost, err = githosts.NewGiteaHost(githosts.NewGiteaHostInput{ - Caller: appName, - APIURL: os.Getenv(envGiteaAPIURL), - DiffRemoteMethod: os.Getenv(envGiteaCompare), - BackupDir: backupDir, - Token: os.Getenv(envGiteaToken), - Orgs: getOrgsListFromEnvVar(envGiteaOrgs), - BackupsToRetain: getBackupsToRetain(envGiteaBackups), - LogLevel: getLogLevel(), - }) - if err != nil { - logger.Fatal(err) - } - - giteaResults := giteaHost.Backup() - - providerBackupResults = append(providerBackupResults, ProviderBackupResults{ - Provider: providerNameBitBucket, - Results: giteaResults, - }) + providerBackupResults = append(providerBackupResults, *Gitea(backupDir)) } if os.Getenv(envGitHubToken) != "" { - logger.Println("backing up GitHub repos") - - var githubHost *githosts.GitHubHost - - githubHost, err = githosts.NewGitHubHost(githosts.NewGitHubHostInput{ - Caller: appName, - APIURL: os.Getenv(envGitHubAPIURL), - DiffRemoteMethod: os.Getenv(envGitHubCompare), - BackupDir: backupDir, - Token: os.Getenv(envGitHubToken), - Orgs: getOrgsListFromEnvVar(envGitHubOrgs), - BackupsToRetain: getBackupsToRetain(envGitHubBackups), - SkipUserRepos: envTrue(envGitHubSkipUserRepos), - LogLevel: getLogLevel(), - }) - if err != nil { - logger.Fatal(err) - } - - githubResults := githubHost.Backup() - providerBackupResults = append(providerBackupResults, ProviderBackupResults{ - Provider: providerNameGitHub, - Results: githubResults, - }) + providerBackupResults = append(providerBackupResults, *GitHub(backupDir)) } if os.Getenv(envGitLabToken) != "" { - logger.Println("backing up GitLab repos") - - var gitlabHost *githosts.GitLabHost - - gitlabHost, err = githosts.NewGitLabHost(githosts.NewGitLabHostInput{ - Caller: appName, - APIURL: os.Getenv(envGitLabAPIURL), - DiffRemoteMethod: os.Getenv(envGitLabCompare), - BackupDir: backupDir, - Token: os.Getenv(envGitLabToken), - BackupsToRetain: getBackupsToRetain(envGitLabBackups), - ProjectMinAccessLevel: getProjectMinimumAccessLevel(), - LogLevel: getLogLevel(), - }) - if err != nil { - logger.Fatal(err) - } - - gitlabResults := gitlabHost.Backup() - - providerBackupResults = append(providerBackupResults, ProviderBackupResults{ - Provider: providerNameGitLab, - Results: gitlabResults, - }) + providerBackupResults = append(providerBackupResults, *Gitlab(backupDir)) } logger.Println("cleaning up") @@ -526,7 +426,6 @@ func execProviderBackups() { backupDir+pathSep+workingDIRName) } - // TODO: use a debug flag to enable this // logger.Printf("file removals took %s", time.Since(startFileRemovals).String()) backupResults.Results = &providerBackupResults @@ -558,6 +457,16 @@ func execProviderBackups() { } } + // help avoid thrashing provider apis if job auto-restartsrestarts + // after an early failure by adding delay if backup took less than 10 seconds + if time.Since(startTime) < time.Second*defaultEarlyErrorBackOffSeconds { + logger.Printf("backup took less than 10 seconds, "+ + "waiting %d seconds before next run to avoid thrashing"+ + " provider apis", defaultEarlyErrorBackOffSeconds) + + time.Sleep(time.Second * defaultEarlyErrorBackOffSeconds) + } + if backupInterval := getBackupInterval(); backupInterval > 0 { nextBackupTime := startTime.Add(time.Duration(backupInterval) * time.Minute) if nextBackupTime.Before(time.Now()) { @@ -568,14 +477,6 @@ func execProviderBackups() { } } -func stripTrailingLineBreak(input string) string { - if strings.HasSuffix(input, "\n") { - return input[:len(input)-2] - } - - return input -} - func getProjectMinimumAccessLevel() int { if os.Getenv(envGitLabMinAccessLevel) == "" { logger.Printf("environment variable %s not set, using default of %d", envGitLabMinAccessLevel, defaultGitLabMinimumProjectAccessLevel) diff --git a/main_test.go b/main_test.go index 1a7dd10..601f46c 100644 --- a/main_test.go +++ b/main_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -280,6 +281,41 @@ func TestPublicGithubRepositoryBackupWithBackupsToKeepUnset(t *testing.T) { require.Len(t, files, 2) } +func TestGithubRepositoryBackupWithInvalidToken(t *testing.T) { + _ = os.Unsetenv(envSobaWebHookURL) + + envBackup := backupEnvironmentVariables() + defer restoreEnvironmentVariables(envBackup) + + preflight() + resetGlobals() + + defer resetBackups() + + // Unset Env Vars but exclude those defined + unsetEnvVarsExcept([]string{envGitBackupDir, envGitHubToken, envGitHubCompare}) + + // set invalid token + _ = os.Setenv(envGitHubToken, "invalid") + + githubHost, err := githosts.NewGitHubHost(githosts.NewGitHubHostInput{ + Caller: appName, + APIURL: os.Getenv(envGitHubAPIURL), + DiffRemoteMethod: os.Getenv(envGitHubCompare), + BackupDir: os.TempDir(), + Token: os.Getenv(envGitHubToken), + Orgs: getOrgsListFromEnvVar(envGitHubOrgs), + BackupsToRetain: getBackupsToRetain(envGitHubBackups), + SkipUserRepos: envTrue(envGitHubSkipUserRepos), + LogLevel: getLogLevel(), + }) + require.NoError(t, err) + + result := githubHost.Backup() + require.NotNil(t, result.Error) + require.Contains(t, errors.Unwrap(result.Error).Error(), "Bad credentials") +} + func TestPublicGithubRepositoryBackup(t *testing.T) { if os.Getenv(envGitHubToken) == "" { t.Skipf("Skipping GitHub test as %s is missing", envGitHubToken) @@ -691,12 +727,15 @@ func TestGithubRepositoryBackupWithWildcardOrgsAndPersonal(t *testing.T) { } result := githubHost.Backup() - require.Len(t, result, 7) + require.Len(t, result.BackupResults, 7) + require.Nil(t, result.Error) - for _, r := range result { + for _, r := range result.BackupResults { require.Nil(t, r.Error) } + require.Nil(t, result.Error) + for _, repoName := range []string{"public1", "public2"} { require.DirExists(t, path.Join(backupDir, "github.com", "Nudelmesse", repoName)) diff --git a/webhook.go b/webhook.go index 0ad192c..c125c03 100644 --- a/webhook.go +++ b/webhook.go @@ -44,10 +44,9 @@ func sendWebhook(c *retryablehttp.Client, sendTime sobaTime, results BackupResul } // send to webhook - client := c - client.RetryMax = 3 - client.RetryWaitMin = 1 * time.Second - client.RetryWaitMax = 3 * time.Second + c.RetryMax = 3 + c.RetryWaitMin = 1 * time.Second + c.RetryWaitMax = 3 * time.Second var req *retryablehttp.Request @@ -58,7 +57,7 @@ func sendWebhook(c *retryablehttp.Client, sendTime sobaTime, results BackupResul req.Header.Set("Content-Type", "application/json") - _, err = client.Do(req) + _, err = c.Do(req) if err != nil { fmt.Printf("error: %s\n", err) } @@ -80,11 +79,11 @@ func (j sobaTime) format() string { return j.Time.Format(j.f) } -func (j sobaTime) MarshalText() ([]byte, error) { +func (j sobaTime) MarshalText() ([]byte, error) { // nolint: unparam return []byte(j.format()), nil } -func (j sobaTime) MarshalJSON() ([]byte, error) { +func (j sobaTime) MarshalJSON() ([]byte, error) { // nolint: unparam return []byte(`"` + j.format() + `"`), nil } @@ -102,7 +101,7 @@ func getBackupsStats(br BackupResults) (ok, failed int) { } for _, pr := range *br.Results { - for _, r := range pr.Results { + for _, r := range pr.Results.BackupResults { if r.Error != nil { failed++ diff --git a/webhook_test.go b/webhook_test.go index ac171e8..7f5bf56 100644 --- a/webhook_test.go +++ b/webhook_test.go @@ -17,16 +17,19 @@ var testProviderBackupResults = []ProviderBackupResults{ { Provider: "GitHub", Results: githosts.ProviderBackupResult{ - githosts.RepoBackupResults{ - Repo: "https://github.com/jonhadfield/githosts-utils", - Status: "ok", - Error: nil, - }, - githosts.RepoBackupResults{ - Repo: "https://github.com/jonhadfield/soba", - Status: "ok", - Error: nil, + BackupResults: []githosts.RepoBackupResults{ + { + Repo: "https://github.com/jonhadfield/githosts-utils", + Status: "ok", + Error: nil, + }, + { + Repo: "https://github.com/jonhadfield/soba", + Status: "ok", + Error: nil, + }, }, + Error: nil, }, }, } @@ -52,7 +55,7 @@ func TestWebhookLongFormat(t *testing.T) { start := theTime.Add(-time.Minute * 20) end := theTime.Add(-time.Second * 10) - json := `{"app":"soba","type":"backups.complete","stats":{"succeeded":2,"failed":0},"timestamp":"2024-01-15T14:30:45Z","data":{"started_at":"2024-01-15T14:10:45Z","finished_at":"2024-01-15T14:30:35Z","results":[{"provider":"GitHub","results":[{"repo":"https://github.com/jonhadfield/githosts-utils","status":"ok"},{"repo":"https://github.com/jonhadfield/soba","status":"ok"}]}]}}` + json := `{"app":"soba","type":"backups.complete","stats":{"succeeded":2,"failed":0},"timestamp":"2024-01-15T14:30:45Z","data":{"started_at":"2024-01-15T14:10:45Z","finished_at":"2024-01-15T14:30:35Z","results":[{"provider":"GitHub","results":{"BackupResults":[{"repo":"https://github.com/jonhadfield/githosts-utils","status":"ok"},{"repo":"https://github.com/jonhadfield/soba","status":"ok"}],"Error":null}}]}}` gock.New(exampleWebHookURL). Post(u.Path). MatchHeader("Content-Type", "application/json").