Skip to content

Commit

Permalink
backup issues from github (#185)
Browse files Browse the repository at this point in the history 
* backup issues from github

* gitlab issues

* added gitea

* added gogs

* added onedev

* updated examples

* added more documentation
  • Loading branch information
@cooperspencer
cooperspencer authored Dec 1, 2023
1 parent a7f72fc commit 52a834b
Show file tree
Hide file tree
Showing 8 changed files with 209 additions and 8 deletions.
5 changes: 5 additions & 0 deletions conf.example.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ source:
- foo1
- bar1
wiki: true # includes wiki too
issues: true # back up issues, works only locally
starred: true # includes the user's starred repositories too
filter:
stars: 100 # only clone repos with 100 stars
Expand Down Expand Up @@ -60,6 +61,7 @@ source:
- foo1
- bar1
wiki: true # includes wiki too
issues: true # back up issues, works only locally
starred: true # includes the user's starred repositories too
filter:
stars: 100 # only clone repos with 100 stars
Expand Down Expand Up @@ -95,6 +97,7 @@ source:
- foo1
- bar1
wiki: true # includes wiki too
issues: true # back up issues, works only locally
filter:
stars: 100 # only clone repos with 100 stars
lastactivity: 1y # only clone repos which had activity during the last year
Expand Down Expand Up @@ -125,6 +128,7 @@ source:
- foo1
- bar1
wiki: true # includes wiki too
issues: true # back up issues, works only locally
starred: true # includes the user's starred repositories too
filter:
stars: 100 # only clone repos with 100 stars
Expand Down Expand Up @@ -163,6 +167,7 @@ source:
filter:
lastactivity: 1y # only clone repos which had activity during the last year
excludeforks: true # exclude forked repositories
issues: true # back up issues, works only locally
sourcehut:
- token: some-token # as of now only the legacy api works, use the legacy token
# token_file: token.txt # alternatively, specify token in a file
Expand Down
29 changes: 29 additions & 0 deletions gitea/gitea.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package gitea

import (
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -312,6 +313,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -347,6 +349,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -455,6 +458,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -490,6 +494,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -523,3 +528,27 @@ func getOrgRepos(client *gitea.Client, org *gitea.Organization,

return o
}

// GetIssues get issues
func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo) map[string]interface{} {
issues := map[string]interface{}{}
if conf.Issues {
listOptions := gitea.ListIssueOption{State: gitea.StateAll, ListOptions: gitea.ListOptions{PageSize: 100}}
for {
i, _, err := client.ListRepoIssues(repo.Owner.UserName, repo.Name, listOptions)
if err != nil {
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
} else {
if len(i) > 0 {
for _, issue := range i {
issues[strconv.Itoa(int(issue.Index))] = issue
}
} else {
break
}
listOptions.Page++
}
}
}
return issues
}
30 changes: 30 additions & 0 deletions github/github.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package github

import (
"context"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -256,6 +257,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: "github.com",
Description: r.GetDescription(),
Private: r.GetPrivate(),
Issues: GetIssues(r, client, repo),
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
Expand Down Expand Up @@ -287,11 +289,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: "github.com",
Description: r.GetDescription(),
Private: r.GetPrivate(),
Issues: GetIssues(r, client, repo),
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
repos = append(repos, wiki)
}

}
} else {
repos = append(repos, types.Repo{
Expand All @@ -305,6 +309,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: "github.com",
Description: r.GetDescription(),
Private: r.GetPrivate(),
Issues: GetIssues(r, client, repo),
})
wiki := addWiki(*r, repo, token)
if wiki.Name != "" {
Expand All @@ -318,6 +323,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
return repos, ran
}

// GetOrCreate Get or create a repository
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com")
token := destination.GetToken()
Expand Down Expand Up @@ -369,3 +375,27 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {

return *r.CloneURL, nil
}

// GetIssues get issues
func GetIssues(repo *github.Repository, client *github.Client, conf types.GenRepo) map[string]interface{} {
issues := map[string]interface{}{}
if conf.Issues {
listOptions := &github.IssueListByRepoOptions{State: "all", ListOptions: github.ListOptions{Page: 0, PerPage: 100}}
for {
i, _, err := client.Issues.ListByRepo(context.Background(), *repo.Owner.Login, *repo.Name, listOptions)
if err != nil {
sub.Error().Err(err).Str("repo", *repo.Name).Msg("can't fetch issues")
} else {
if len(i) > 0 {
for _, issue := range i {
issues[strconv.Itoa(*issue.Number)] = issue
}
} else {
break
}
listOptions.Page++
}
}
}
return issues
}
35 changes: 32 additions & 3 deletions gitlab/gitlab.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package gitlab
import (
"fmt"
"path"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -100,16 +101,16 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gitlab {
if repo.URL == "" {
repo.URL = "https://gitlab.com"
}
err := repo.Filter.ParseDuration()
sub = logger.CreateSubLogger("stage", "gitlab", "url", repo.URL)
if err != nil {
sub.Error().
Msg(err.Error())
}
ran = true
if repo.URL == "" {
repo.URL = "https://gitlab.com"
}

sub.Info().
Msgf("grabbing repositories from %s", repo.User)
Expand Down Expand Up @@ -230,6 +231,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Visibility == gitlab.PrivateVisibility,
Issues: GetIssues(r, client, repo),
})
}

Expand Down Expand Up @@ -270,6 +272,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Visibility == gitlab.PrivateVisibility,
Issues: GetIssues(r, client, repo),
})
}

Expand Down Expand Up @@ -395,6 +398,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Visibility == gitlab.PrivateVisibility,
Issues: GetIssues(r, client, repo),
})
}

Expand Down Expand Up @@ -439,6 +443,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Visibility == gitlab.PrivateVisibility,
Issues: GetIssues(r, client, repo),
})
}

Expand Down Expand Up @@ -483,3 +488,27 @@ func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bo

return len(wikis) > 0
}

// GetIssues get issues
func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) map[string]interface{} {
issues := map[string]interface{}{}
if conf.Issues {
listOptions := &gitlab.ListProjectIssuesOptions{ListOptions: gitlab.ListOptions{PerPage: 100}}
for {
i, _, err := client.Issues.ListProjectIssues(repo.ID, listOptions)
if err != nil {
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
} else {
if len(i) > 0 {
for _, issue := range i {
issues[strconv.Itoa(issue.IID)] = issue
}
} else {
break
}
listOptions.Page++
}
}
}
return issues
}
29 changes: 29 additions & 0 deletions gogs/gogs.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package gogs

import (
"strconv"
"time"

"github.com/cooperspencer/gickup/logger"
Expand Down Expand Up @@ -194,6 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if repo.Wiki {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -229,6 +231,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if repo.Wiki {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -319,6 +322,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})
if repo.Wiki {
repos = append(repos, types.Repo{
Expand Down Expand Up @@ -354,6 +358,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
Hoster: types.GetHost(repo.URL),
Description: r.Description,
Private: r.Private,
Issues: GetIssues(r, client, repo),
})

if repo.Wiki {
Expand All @@ -376,3 +381,27 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {

return repos, ran
}

// GetIssues get issues
func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) map[string]interface{} {
issues := map[string]interface{}{}
if conf.Issues {
listOptions := gogs.ListIssueOption{State: "all"}
for {
i, err := client.ListRepoIssues(repo.Owner.UserName, repo.Name, listOptions)
if err != nil {
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
} else {
if len(i) > 0 {
for _, issue := range i {
issues[strconv.Itoa(int(issue.Index))] = issue
}
} else {
break
}
listOptions.Page++
}
}
}
return issues
}
Loading

0 comments on commit 52a834b

Please sign in to comment.