diff --git a/conf.example.yml b/conf.example.yml index 6b10ae4..4d6aaa4 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -25,6 +25,7 @@ source: - foo1 - bar1 wiki: true # includes wiki too + issues: true # back up issues, works only locally starred: true # includes the user's starred repositories too filter: stars: 100 # only clone repos with 100 stars @@ -60,6 +61,7 @@ source: - foo1 - bar1 wiki: true # includes wiki too + issues: true # back up issues, works only locally starred: true # includes the user's starred repositories too filter: stars: 100 # only clone repos with 100 stars @@ -95,6 +97,7 @@ source: - foo1 - bar1 wiki: true # includes wiki too + issues: true # back up issues, works only locally filter: stars: 100 # only clone repos with 100 stars lastactivity: 1y # only clone repos which had activity during the last year @@ -125,6 +128,7 @@ source: - foo1 - bar1 wiki: true # includes wiki too + issues: true # back up issues, works only locally starred: true # includes the user's starred repositories too filter: stars: 100 # only clone repos with 100 stars @@ -163,6 +167,7 @@ source: filter: lastactivity: 1y # only clone repos which had activity during the last year excludeforks: true # exclude forked repositories + issues: true # back up issues, works only locally sourcehut: - token: some-token # as of now only the legacy api works, use the legacy token # token_file: token.txt # alternatively, specify token in a file diff --git a/gitea/gitea.go b/gitea/gitea.go index 5b794de..fe2351a 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -1,6 +1,7 @@ package gitea import ( + "strconv" "strings" "time" @@ -312,6 +313,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { repos = append(repos, types.Repo{ @@ -347,6 +349,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { repos = append(repos, types.Repo{ @@ -455,6 +458,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { repos = append(repos, types.Repo{ @@ -490,6 +494,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) { repos = append(repos, types.Repo{ @@ -523,3 +528,27 @@ func getOrgRepos(client *gitea.Client, org *gitea.Organization, return o } + +// GetIssues get issues +func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo) map[string]interface{} { + issues := map[string]interface{}{} + if conf.Issues { + listOptions := gitea.ListIssueOption{State: gitea.StateAll, ListOptions: gitea.ListOptions{PageSize: 100}} + for { + i, _, err := client.ListRepoIssues(repo.Owner.UserName, repo.Name, listOptions) + if err != nil { + sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues") + } else { + if len(i) > 0 { + for _, issue := range i { + issues[strconv.Itoa(int(issue.Index))] = issue + } + } else { + break + } + listOptions.Page++ + } + } + } + return issues +} diff --git a/github/github.go b/github/github.go index 2a92c20..2747eec 100644 --- a/github/github.go +++ b/github/github.go @@ -2,6 +2,7 @@ package github import ( "context" + "strconv" "strings" "time" @@ -256,6 +257,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: "github.com", Description: r.GetDescription(), Private: r.GetPrivate(), + Issues: GetIssues(r, client, repo), }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { @@ -287,11 +289,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: "github.com", Description: r.GetDescription(), Private: r.GetPrivate(), + Issues: GetIssues(r, client, repo), }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { repos = append(repos, wiki) } + } } else { repos = append(repos, types.Repo{ @@ -305,6 +309,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: "github.com", Description: r.GetDescription(), Private: r.GetPrivate(), + Issues: GetIssues(r, client, repo), }) wiki := addWiki(*r, repo, token) if wiki.Name != "" { @@ -318,6 +323,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { return repos, ran } +// GetOrCreate Get or create a repository func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") token := destination.GetToken() @@ -369,3 +375,27 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { return *r.CloneURL, nil } + +// GetIssues get issues +func GetIssues(repo *github.Repository, client *github.Client, conf types.GenRepo) map[string]interface{} { + issues := map[string]interface{}{} + if conf.Issues { + listOptions := &github.IssueListByRepoOptions{State: "all", ListOptions: github.ListOptions{Page: 0, PerPage: 100}} + for { + i, _, err := client.Issues.ListByRepo(context.Background(), *repo.Owner.Login, *repo.Name, listOptions) + if err != nil { + sub.Error().Err(err).Str("repo", *repo.Name).Msg("can't fetch issues") + } else { + if len(i) > 0 { + for _, issue := range i { + issues[strconv.Itoa(*issue.Number)] = issue + } + } else { + break + } + listOptions.Page++ + } + } + } + return issues +} diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index 9c3b574..05fbbd5 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -3,6 +3,7 @@ package gitlab import ( "fmt" "path" + "strconv" "strings" "time" @@ -100,6 +101,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gitlab { + if repo.URL == "" { + repo.URL = "https://gitlab.com" + } err := repo.Filter.ParseDuration() sub = logger.CreateSubLogger("stage", "gitlab", "url", repo.URL) if err != nil { @@ -107,9 +111,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Msg(err.Error()) } ran = true - if repo.URL == "" { - repo.URL = "https://gitlab.com" - } sub.Info(). Msgf("grabbing repositories from %s", repo.User) @@ -230,6 +231,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } @@ -270,6 +272,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } @@ -395,6 +398,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } @@ -439,6 +443,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } @@ -483,3 +488,27 @@ func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bo return len(wikis) > 0 } + +// GetIssues get issues +func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) map[string]interface{} { + issues := map[string]interface{}{} + if conf.Issues { + listOptions := &gitlab.ListProjectIssuesOptions{ListOptions: gitlab.ListOptions{PerPage: 100}} + for { + i, _, err := client.Issues.ListProjectIssues(repo.ID, listOptions) + if err != nil { + sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues") + } else { + if len(i) > 0 { + for _, issue := range i { + issues[strconv.Itoa(issue.IID)] = issue + } + } else { + break + } + listOptions.Page++ + } + } + } + return issues +} diff --git a/gogs/gogs.go b/gogs/gogs.go index bbae1e4..e72afcb 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -1,6 +1,7 @@ package gogs import ( + "strconv" "time" "github.com/cooperspencer/gickup/logger" @@ -194,6 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if repo.Wiki { repos = append(repos, types.Repo{ @@ -229,6 +231,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if repo.Wiki { repos = append(repos, types.Repo{ @@ -319,6 +322,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if repo.Wiki { repos = append(repos, types.Repo{ @@ -354,6 +358,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Private, + Issues: GetIssues(r, client, repo), }) if repo.Wiki { @@ -376,3 +381,27 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { return repos, ran } + +// GetIssues get issues +func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) map[string]interface{} { + issues := map[string]interface{}{} + if conf.Issues { + listOptions := gogs.ListIssueOption{State: "all"} + for { + i, err := client.ListRepoIssues(repo.Owner.UserName, repo.Name, listOptions) + if err != nil { + sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues") + } else { + if len(i) > 0 { + for _, issue := range i { + issues[strconv.Itoa(int(issue.Index))] = issue + } + } else { + break + } + listOptions.Page++ + } + } + } + return issues +} diff --git a/local/local.go b/local/local.go index dc9468c..f5f3cbf 100644 --- a/local/local.go +++ b/local/local.go @@ -1,11 +1,13 @@ package local import ( + "encoding/json" "fmt" "math/rand" "net" "os" "path" + "path/filepath" "sort" "strconv" "strings" @@ -182,18 +184,51 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } } + if len(repo.Issues) > 0 { + _, err := os.Stat(fmt.Sprintf("%s.issues", repo.Name)) + if os.IsNotExist(err) && !dry { + if err := os.MkdirAll(fmt.Sprintf("%s.issues", repo.Name), 0o777); err != nil { + sub.Error(). + Msg(err.Error()) + } + } + sub.Info().Str("repo", repo.Name).Msg("backing up issues") + if !dry { + for k, v := range repo.Issues { + jsonData, err := json.Marshal(v) + if err != nil { + sub.Error(). + Msg(err.Error()) + } else { + err = os.WriteFile(filepath.Join(l.Path, fmt.Sprintf("%s.issues", repo.Name), fmt.Sprintf("%s.json", k)), jsonData, 0644) + if err != nil { + sub.Error(). + Msg(err.Error()) + } + } + } + } + } + if l.Zip { + tozip := []string{repo.Name} + + if len(repo.Issues) > 0 { + tozip = append(tozip, fmt.Sprintf("%s.issues", repo.Name)) + } sub.Info(). Msgf("zipping %s", types.Green(repo.Name)) - err := archiver.Archive([]string{repo.Name}, fmt.Sprintf("%s.zip", repo.Name)) + err := archiver.Archive(tozip, fmt.Sprintf("%s.zip", repo.Name)) if err != nil { sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) } - err = os.RemoveAll(repo.Name) - if err != nil { - sub.Warn(). - Str("repo", repo.Name).Msg(err.Error()) + for _, dir := range tozip { + err = os.RemoveAll(dir) + if err != nil { + sub.Warn(). + Str("repo", repo.Name).Msg(err.Error()) + } } } diff --git a/onedev/onedev.go b/onedev/onedev.go index 270a43c..f718b12 100644 --- a/onedev/onedev.go +++ b/onedev/onedev.go @@ -2,6 +2,8 @@ package onedev import ( "fmt" + "strconv" + "strings" "time" "github.com/cooperspencer/gickup/logger" @@ -135,6 +137,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Owner: repo.User, Hoster: types.GetHost(repo.URL), Description: r.Description, + Issues: GetIssues(&r, client, repo, urls.HTTP), }) } @@ -197,6 +200,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Owner: org, Hoster: types.GetHost(repo.URL), Description: r.Description, + Issues: GetIssues(&r, client, repo, urls.HTTP), }) } } @@ -282,3 +286,41 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { return cloneUrls.HTTP, nil } + +// GetIssues get issues +func GetIssues(repo *onedev.Project, client *onedev.Client, conf types.GenRepo, repourl string) map[string]interface{} { + issues := map[string]interface{}{} + if conf.Issues { + name := strings.TrimPrefix(repourl, conf.URL) + listOptions := &onedev.IssueQueryOptions{Count: 100, Offset: 0, Query: fmt.Sprintf("\"Project\" is \"%s\"", name)} + for { + i, _, err := client.GetIssues(listOptions) + if err != nil { + sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues") + } else { + if len(i) > 0 { + for _, issue := range i { + onedevissue := Issue{Issue: issue} + comments, _, err := client.GetIssueComments(onedevissue.ID) + if err != nil { + sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues") + } else { + onedevissue.Comments = comments + } + + issues[strconv.Itoa(int(issue.Number))] = onedevissue + } + } else { + break + } + listOptions.Offset += listOptions.Count + } + } + } + return issues +} + +type Issue struct { + onedev.Issue + Comments []onedev.Comment +} diff --git a/types/types.go b/types/types.go index 2ec9aa7..1421047 100644 --- a/types/types.go +++ b/types/types.go @@ -241,6 +241,7 @@ type GenRepo struct { ExcludeOrgs []string `yaml:"excludeorgs"` Include []string `yaml:"include"` IncludeOrgs []string `yaml:"includeorgs"` + Issues bool `yaml:"issues"` Wiki bool `yaml:"wiki"` Starred bool `yaml:"starred"` CreateOrg bool `yaml:"createorg"` @@ -384,6 +385,7 @@ type Repo struct { Owner string Hoster string Description string + Issues map[string]interface{} Private bool }