From f61574bf9cbc10fa21a8326df55ac2fb4d7517b7 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Fri, 10 Feb 2023 06:07:11 +0100 Subject: [PATCH] improved prometheus metrics (#134) * improved prometheus metrics * change RepoTime to gauge * fix gitea, only set metrics if ran --- bitbucket/bitbucket.go | 6 ++- gitea/gitea.go | 57 +++++++++++++--------- github/github.go | 6 ++- gitlab/gitlab.go | 21 +++++--- gogs/gogs.go | 19 +++++--- local/local.go | 15 ++++-- main.go | 84 ++++++++++++++++++++++++-------- metrics/prometheus/prometheus.go | 10 ++++ onedev/onedev.go | 6 ++- sourcehut/sourcehut.go | 6 ++- whatever/whatever.go | 6 ++- 11 files changed, 165 insertions(+), 71 deletions(-) diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index c06147a..efb1dfb 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -9,9 +9,11 @@ import ( ) // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.BitBucket { + ran = true client := bitbucket.NewBasicAuth(repo.Username, repo.Password) if repo.User == "" { repo.User = repo.Username @@ -88,5 +90,5 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } diff --git a/gitea/gitea.go b/gitea/gitea.go index 36d8173..e6801fd 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -31,7 +31,7 @@ func getRepoVisibility(visibility string) bool { } // Backup TODO. -func Backup(r types.Repo, d types.GenRepo, dry bool) { +func Backup(r types.Repo, d types.GenRepo, dry bool) bool { orgvisibilty := getOrgVisibility(d.Visibility.Organizations) repovisibility := getRepoVisibility(d.Visibility.Repositories) if d.URL == "" { @@ -45,15 +45,17 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken())) if err != nil { - log.Fatal().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error()) + log.Error().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error()) + return false } user, _, err := giteaclient.GetMyUserInfo() if err != nil { - log.Fatal(). + log.Error(). Str("stage", "gitea"). Str("url", d.URL). Msg(err.Error()) + return false } if d.User == "" && d.CreateOrg { @@ -69,25 +71,27 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Visibility: orgvisibilty, }) if err != nil { - log.Fatal(). + log.Error(). Str("stage", "gitea"). Str("url", d.URL). Msg(err.Error()) + return false } user.ID = org.ID user.UserName = org.UserName } else { - log.Fatal(). + log.Error(). Str("stage", "gitea"). Str("url", d.URL). Msg(err.Error()) + return false } } } if dry { - return + return true } repo, _, err := giteaclient.GetRepo(user.UserName, r.Name) @@ -121,7 +125,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("stage", "gitea"). Str("url", d.URL). Err(err) - return + return false } log.Info(). @@ -129,7 +133,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("url", d.URL). Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL) - return + return true } if repo.Mirror { log.Info(). @@ -143,7 +147,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("stage", "gitea"). Str("url", d.URL). Err(err) - return + return false } log.Info(). @@ -151,12 +155,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("url", d.URL). Msgf("successfully synced %s.", types.Blue(r.Name)) } + + return true } // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gitea { + ran = true if repo.URL == "" { repo.URL = "https://gitea.com" } @@ -293,21 +301,22 @@ func Get(conf *types.Conf) []types.Repo { } orgopt := gitea.ListOptions{Page: 1, PageSize: 50} orgs := []*gitea.Organization{} - for { - o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt}) - if err != nil { - log.Fatal(). - Str("stage", "gitea"). - Str("url", repo.URL). - Msg(err.Error()) - } - if len(o) == 0 { - break + if token != "" { + for { + o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt}) + if err != nil { + log.Fatal(). + Str("stage", "gitea"). + Str("url", repo.URL). + Msg(err.Error()) + } + if len(o) == 0 { + break + } + orgs = append(orgs, o...) + orgopt.Page++ } - orgs = append(orgs, o...) - orgopt.Page++ } - orgrepos := []*gitea.Repository{} for _, org := range orgs { orgopt.Page = 1 @@ -392,7 +401,7 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } func getOrgRepos(client *gitea.Client, org *gitea.Organization, diff --git a/github/github.go b/github/github.go index 5466943..117b3bf 100644 --- a/github/github.go +++ b/github/github.go @@ -28,9 +28,11 @@ func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { } // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.Github { + ran = true if repo.User == "" { log.Info(). Str("stage", "github"). @@ -194,5 +196,5 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index 55a3b4e..e127740 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -11,7 +11,7 @@ import ( ) // Backup TODO. -func Backup(r types.Repo, d types.GenRepo, dry bool) { +func Backup(r types.Repo, d types.GenRepo, dry bool) bool { var gitlabclient *gitlab.Client token := d.GetToken() var err error @@ -23,10 +23,11 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { } if err != nil { - log.Fatal(). + log.Error(). Str("stage", "gitlab"). Str("url", d.URL). Msg(err.Error()) + return false } log.Info(). @@ -43,7 +44,8 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { projects, _, err := gitlabclient.Projects.ListProjects(&opt) if err != nil { - log.Fatal().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error()) + log.Error().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error()) + return false } found := false @@ -54,7 +56,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { } if dry || found { - return + return true } if r.Token != "" { @@ -72,17 +74,22 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { _, _, err = gitlabclient.Projects.CreateProject(opts) if err != nil { - log.Fatal(). + log.Error(). Str("stage", "gitlab"). Str("url", d.URL). Msg(err.Error()) + return false } + + return true } // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gitlab { + ran = true if repo.URL == "" { repo.URL = "https://gitlab.com" } @@ -360,7 +367,7 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bool { diff --git a/gogs/gogs.go b/gogs/gogs.go index e6e3fc8..7b7593b 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -18,7 +18,7 @@ func getRepoVisibility(visibility string) bool { } // Backup TODO. -func Backup(r types.Repo, d types.GenRepo, dry bool) { +func Backup(r types.Repo, d types.GenRepo, dry bool) bool { repovisibility := getRepoVisibility(d.Visibility.Repositories) log.Info(). Str("stage", "gogs"). @@ -29,10 +29,11 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { user, err := gogsclient.GetSelfInfo() if err != nil { - log.Fatal(). + log.Error(). Str("stage", "gogs"). Str("url", d.URL). Msg(err.Error()) + return false } if d.User == "" && d.CreateOrg { @@ -64,7 +65,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { } if dry { - return + return true } repo, err := gogsclient.GetRepo(user.UserName, r.Name) @@ -96,9 +97,10 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("stage", "gogs"). Str("url", d.URL). Err(err) + return false } - return + return true } if repo.Mirror { @@ -113,6 +115,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("stage", "gogs"). Str("url", d.URL). Err(err) + return false } log.Info(). @@ -120,12 +123,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) { Str("url", d.URL). Msgf("successfully synced %s.", types.Blue(r.Name)) } + + return true } // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gogs { + ran = true if repo.User == "" { log.Info(). Str("stage", "gogs"). @@ -331,5 +338,5 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } diff --git a/local/local.go b/local/local.go index 6fe18dd..0be54ea 100644 --- a/local/local.go +++ b/local/local.go @@ -24,7 +24,7 @@ import ( ) // Locally TODO. -func Locally(repo types.Repo, l types.Local, dry bool) { +func Locally(repo types.Repo, l types.Local, dry bool) bool { date := time.Now() if l.Structured { @@ -42,10 +42,11 @@ func Locally(repo types.Repo, l types.Local, dry bool) { stat, err := os.Stat(l.Path) if os.IsNotExist(err) && !dry { if err := os.MkdirAll(l.Path, 0o777); err != nil { - log.Fatal(). + log.Error(). Str("stage", "locally"). Str("path", l.Path). Msg(err.Error()) + return false } stat, _ = os.Stat(l.Path) @@ -53,10 +54,11 @@ func Locally(repo types.Repo, l types.Local, dry bool) { if stat != nil && stat.IsDir() { if err := os.Chdir(l.Path); err != nil { - log.Fatal(). + log.Error(). Str("stage", "locally"). Str("path", l.Path). Msg(err.Error()) + return false } } @@ -73,7 +75,11 @@ func Locally(repo types.Repo, l types.Local, dry bool) { auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "") if err != nil { - panic(err) + log.Error(). + Str("stage", "locally"). + Str("path", l.Path). + Msg(err.Error()) + return false } case repo.Token != "": auth = &http.BasicAuth{ @@ -262,6 +268,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) { x = 5 } + return true } func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, bare bool) error { diff --git a/main.go b/main.go index 7ffc7a9..40ca3fb 100644 --- a/main.go +++ b/main.go @@ -153,27 +153,55 @@ func backup(repos []types.Repo, conf *types.Conf) { checkedpath = true } - local.Locally(r, d, cli.Dry) + repotime := time.Now() + status := 0 + if local.Locally(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "local", d.Path).Set(time.Now().Sub(repotime).Seconds()) + status = 1 + } + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "local", d.Path).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() } for _, d := range conf.Destination.Gitea { if !strings.HasSuffix(r.Name, ".wiki") { - gitea.Backup(r, d, cli.Dry) + repotime := time.Now() + status := 0 + if gitea.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Now().Sub(repotime).Seconds()) + status = 1 + } + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() } } for _, d := range conf.Destination.Gogs { if !strings.HasSuffix(r.Name, ".wiki") { - gogs.Backup(r, d, cli.Dry) + repotime := time.Now() + status := 0 + if gogs.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Now().Sub(repotime).Seconds()) + status = 1 + } + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() } } for _, d := range conf.Destination.Gitlab { if !strings.HasSuffix(r.Name, ".wiki") { - gitlab.Backup(r, d, cli.Dry) + repotime := time.Now() + status := 0 + if gitlab.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitlab", d.URL).Set(time.Now().Sub(repotime).Seconds()) + status = 1 + } + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitlab", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc() } } @@ -192,39 +220,55 @@ func runBackup(conf *types.Conf, num int) { prometheus.JobsStarted.Inc() // Github - repos := github.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("github", numstring).Set(float64(len(repos))) + repos, ran := github.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("github", numstring).Set(float64(len(repos))) + } backup(repos, conf) // Gitea - repos = gitea.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("gitea", numstring).Set(float64(len(repos))) + repos, ran = gitea.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("gitea", numstring).Set(float64(len(repos))) + } backup(repos, conf) // Gogs - repos = gogs.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("gogs", numstring).Set(float64(len(repos))) + repos, ran = gogs.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("gogs", numstring).Set(float64(len(repos))) + } backup(repos, conf) // Gitlab - repos = gitlab.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("gitlab", numstring).Set(float64(len(repos))) + repos, ran = gitlab.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("gitlab", numstring).Set(float64(len(repos))) + } backup(repos, conf) - repos = bitbucket.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("bitbucket", numstring).Set(float64(len(repos))) + repos, ran = bitbucket.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("bitbucket", numstring).Set(float64(len(repos))) + } backup(repos, conf) - repos = whatever.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("whatever", numstring).Set(float64(len(repos))) + repos, ran = whatever.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("whatever", numstring).Set(float64(len(repos))) + } backup(repos, conf) - repos = onedev.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("onedev", numstring).Set(float64(len(repos))) + repos, ran = onedev.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("onedev", numstring).Set(float64(len(repos))) + } backup(repos, conf) - repos = sourcehut.Get(conf) - prometheus.CountReposDiscovered.WithLabelValues("sourcehut", numstring).Set(float64(len(repos))) + repos, ran = sourcehut.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("sourcehut", numstring).Set(float64(len(repos))) + } backup(repos, conf) endTime := time.Now() diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 776f997..18d4da1 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -50,6 +50,16 @@ var DestinationBackupsComplete = promauto.NewCounterVec(prometheus.CounterOpts{ Help: "The count of destination to which a backup was written", }, []string{"destination_type"}) +var RepoSuccess = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gickup_repo_success", + Help: "See if backup was successful", +}, []string{"hoster", "repository", "owner", "type", "path"}) + +var RepoTime = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "gickup_repo_time", + Help: "How long did the task take", +}, []string{"hoster", "repository", "owner", "type", "path"}) + func Serve(conf types.PrometheusConfig) { log.Info(). Str("listenAddr", conf.ListenAddr). diff --git a/onedev/onedev.go b/onedev/onedev.go index 5ddff63..5efdcb0 100644 --- a/onedev/onedev.go +++ b/onedev/onedev.go @@ -8,10 +8,12 @@ import ( "github.com/rs/zerolog/log" ) -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.OneDev { + ran = true if repo.URL == "" { repo.URL = "https://code.onedev.io/" } @@ -182,5 +184,5 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } diff --git a/sourcehut/sourcehut.go b/sourcehut/sourcehut.go index b26b65e..eccbef3 100644 --- a/sourcehut/sourcehut.go +++ b/sourcehut/sourcehut.go @@ -100,9 +100,11 @@ func getRefs(url, name, token string) (Refs, error) { } // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} for _, repo := range conf.Source.Sourcehut { + ran = true if repo.URL == "" { repo.URL = "https://git.sr.ht" } @@ -251,5 +253,5 @@ func Get(conf *types.Conf) []types.Repo { } } - return repos + return repos, ran } diff --git a/whatever/whatever.go b/whatever/whatever.go index 8bdc27e..ea77e66 100644 --- a/whatever/whatever.go +++ b/whatever/whatever.go @@ -15,9 +15,11 @@ import ( ) // Get TODO. -func Get(conf *types.Conf) []types.Repo { +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false repos := []types.Repo{} if len(conf.Source.Any) > 0 { + ran = true log.Info(). Str("stage", "whatever"). Msgf("adding repos") @@ -100,5 +102,5 @@ func Get(conf *types.Conf) []types.Repo { }) } } - return repos + return repos, ran }