From a659ca3d2648338d084acf6bcbfad7dcd2f915da Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Fri, 12 Apr 2024 10:30:26 +0200 Subject: [PATCH] implemented mirror using gickup and not giteas mirror function --- gitea/gitea.go | 81 +++++++++++++++++++++++++++++++++++++++++++++--- main.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++-- types/types.go | 1 + 3 files changed, 158 insertions(+), 7 deletions(-) diff --git a/gitea/gitea.go b/gitea/gitea.go index 4bd21de..43c520e 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -41,6 +41,9 @@ func getRepoVisibility(visibility string, private bool) bool { // Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) bool { + if d.MirrorInterval == "" { + d.MirrorInterval = "8h0m0s" + } orgvisibilty := getOrgVisibility(d.Visibility.Organizations) repovisibility := getRepoVisibility(d.Visibility.Repositories, r.Private) if d.URL == "" { @@ -148,10 +151,12 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { return true } - _, err = time.ParseDuration(d.MirrorInterval) - if err != nil { - sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval) - d.MirrorInterval = repo.MirrorInterval + if d.MirrorInterval != "" { + _, err = time.ParseDuration(d.MirrorInterval) + if err != nil { + sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval) + d.MirrorInterval = repo.MirrorInterval + } } if d.MirrorInterval != repo.MirrorInterval { @@ -559,3 +564,71 @@ func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo) } return issues } + +// GetOrCreate Get or create a repository +func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { + orgvisibilty := getOrgVisibility(destination.Visibility.Organizations) + repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private) + if destination.URL == "" { + destination.URL = "https://gitea.com/" + } + sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL) + + giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken())) + if err != nil { + return "", err + } + + user, _, err := giteaclient.GetMyUserInfo() + if err != nil { + return "", err + } + me := user + + if destination.User == "" && destination.CreateOrg { + destination.User = repo.Owner + } + + if destination.User != "" { + user, _, err = giteaclient.GetUserInfo(destination.User) + if err != nil { + if destination.CreateOrg { + org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{ + Name: destination.User, + Visibility: orgvisibilty, + }) + if err != nil { + return "", err + } + user.ID = org.ID + user.UserName = org.UserName + } else { + return "", err + } + } + + } + + r, _, err := giteaclient.GetRepo(user.UserName, repo.Name) + if err != nil { + opts := gitea.CreateRepoOption{ + Name: repo.Name, + Private: repovisibility, + Description: repo.Description, + } + + if me.UserName == user.UserName { + r, _, err = giteaclient.CreateRepo(opts) + if err != nil { + return "", err + } + } else { + r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts) + if err != nil { + return "", err + } + } + } + + return r.CloneURL, nil +} diff --git a/main.go b/main.go index 1059424..f51bfad 100644 --- a/main.go +++ b/main.go @@ -187,9 +187,85 @@ func backup(repos []types.Repo, conf *types.Conf) { if !strings.HasSuffix(r.Name, ".wiki") { repotime := time.Now() status := 0 - if gitea.Backup(r, d, cli.Dry) { - prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) - status = 1 + if d.SelfMirror { + if !strings.HasSuffix(r.Name, ".wiki") { + repotime := time.Now() + status := 0 + + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + + if !cli.Dry { + tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gitea-%x", repotime)) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + continue + } + + defer os.RemoveAll(tempdir) + temprepo, err := local.TempClone(r, tempdir) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + cloneurl, err := gitea.GetOrCreate(d, r) + if err != nil { + log.Error(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "gitea"). + Str("url", r.URL). + Str("git", "push"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() + } + } + } else { + if gitea.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + } } prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) @@ -609,6 +685,7 @@ func main() { return } + zerolog.SetGlobalLevel(zerolog.InfoLevel) if cli.Quiet { zerolog.SetGlobalLevel(zerolog.WarnLevel) } diff --git a/types/types.go b/types/types.go index 1421047..ec48565 100644 --- a/types/types.go +++ b/types/types.go @@ -251,6 +251,7 @@ type GenRepo struct { Contributed bool `yaml:"contributed"` MirrorInterval string `yaml:"mirrorinterval"` LFS bool `yaml:"lfs"` + SelfMirror bool `yaml:"selfmirror"` } // Visibility struct