From 27063df008f4b55660c98d977359aed3aa569a18 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 28 Aug 2024 13:10:04 +0200 Subject: [PATCH 1/3] improve debug messages for gitlab --- gitlab/gitlab.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index 1a2252a..fe8f984 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -218,6 +218,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } for _, r := range gitlabrepos { + sub.Debug().Str("repo-type", "user").Msg(r.WebURL) if repo.Filter.ExcludeForks { if r.ForkedFromProject != nil { continue @@ -230,7 +231,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } if len(repo.Filter.Languages) > 0 { - sub.Debug().Msg(r.WebURL) langs, _, err := client.Projects.GetProjectLanguages(r.ID) if err != nil { sub.Error(). @@ -385,6 +385,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } for k, gr := range gitlabgrouprepos { for _, r := range gr { + sub.Debug().Str("repo-type", "group").Msg(r.WebURL) if repo.Filter.ExcludeForks { if r.ForkedFromProject != nil { continue From 4bf2ceae6f818a08c2598ff403839db22af45b58 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 28 Aug 2024 15:56:39 +0200 Subject: [PATCH 2/3] get all repos you have access to --- gitlab/gitlab.go | 331 ++++++++++++++++++++++++----------------------- 1 file changed, 169 insertions(+), 162 deletions(-) diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index fe8f984..815d4b9 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -101,6 +101,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} + inSlice := map[string]bool{} for _, repo := range conf.Source.Gitlab { if repo.URL == "" { repo.URL = "https://gitlab.com" @@ -136,7 +137,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { gitlabrepos := []*gitlab.Project{} gitlabgrouprepos := map[string][]*gitlab.Project{} - opt := &gitlab.ListProjectsOptions{} + opt := &gitlab.ListProjectsOptions{Membership: gitlab.Ptr(true)} users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User}) if err != nil { sub.Error(). @@ -150,7 +151,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { i := 1 for { opt.Page = i - projects, _, err := client.Projects.ListUserProjects(user.ID, opt) + projects, _, err := client.Projects.ListProjects(opt) if err != nil { sub.Error(). Msg(err.Error()) @@ -218,70 +219,52 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } for _, r := range gitlabrepos { - sub.Debug().Str("repo-type", "user").Msg(r.WebURL) - if repo.Filter.ExcludeForks { - if r.ForkedFromProject != nil { - continue + if !inSlice[r.PathWithNamespace] { + sub.Debug().Str("repo-type", "user").Msg(r.WebURL) + if repo.Filter.ExcludeForks { + if r.ForkedFromProject != nil { + continue + } } - } - if repo.Filter.ExcludeArchived { - if r.Archived { - continue + if repo.Filter.ExcludeArchived { + if r.Archived { + continue + } } - } - if len(repo.Filter.Languages) > 0 { - langs, _, err := client.Projects.GetProjectLanguages(r.ID) - if err != nil { - sub.Error(). - Msg(err.Error()) - continue - } else { - language := "" - percentage := float32(0) + if len(repo.Filter.Languages) > 0 { + langs, _, err := client.Projects.GetProjectLanguages(r.ID) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } else { + language := "" + percentage := float32(0) - for lang, percent := range *langs { - if percent > percentage { - language = lang + for lang, percent := range *langs { + if percent > percentage { + language = lang + } + } + if !languages[strings.ToLower(language)] { + continue } - } - if !languages[strings.ToLower(language)] { - continue } } - } - if r.StarCount < repo.Filter.Stars { - continue - } - if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { - continue - } - if include[r.Name] { - if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{ - Name: r.Path, - URL: r.HTTPURLToRepo, - SSHURL: r.SSHURLToRepo, - Token: token, - Defaultbranch: r.DefaultBranch, - Origin: repo, - Owner: r.Namespace.FullPath, - Hoster: types.GetHost(repo.URL), - Description: r.Description, - Private: r.Visibility == gitlab.PrivateVisibility, - Issues: GetIssues(r, client, repo), - }) + if r.StarCount < repo.Filter.Stars { + continue } - - if r.WikiEnabled && repo.Wiki { - if activeWiki(r, client, repo) { - httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { + continue + } + if include[r.Name] { + if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ - Name: r.Path + ".wiki", - URL: httpURLToRepo, - SSHURL: sshURLToRepo, + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, @@ -289,40 +272,40 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } - } - continue - } - if exclude[r.Name] { - continue - } - if len(include) == 0 { - if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{ - Name: r.Path, - URL: r.HTTPURLToRepo, - SSHURL: r.SSHURLToRepo, - Token: token, - Defaultbranch: r.DefaultBranch, - Origin: repo, - Owner: r.Namespace.FullPath, - Hoster: types.GetHost(repo.URL), - Description: r.Description, - Private: r.Visibility == gitlab.PrivateVisibility, - Issues: GetIssues(r, client, repo), - }) - } + if r.WikiEnabled && repo.Wiki { + if activeWiki(r, client, repo) { + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + Description: r.Description, + Private: r.Visibility == gitlab.PrivateVisibility, + }) + } + } - if r.WikiEnabled && repo.Wiki { - if activeWiki(r, client, repo) { - httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + continue + } + if exclude[r.Name] { + continue + } + if len(include) == 0 { + if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ - Name: r.Path + ".wiki", - URL: httpURLToRepo, - SSHURL: sshURLToRepo, + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, @@ -330,9 +313,30 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Hoster: types.GetHost(repo.URL), Description: r.Description, Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), }) } + + if r.WikiEnabled && repo.Wiki { + if activeWiki(r, client, repo) { + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: r.Namespace.FullPath, + Hoster: types.GetHost(repo.URL), + Description: r.Description, + Private: r.Visibility == gitlab.PrivateVisibility, + }) + } + } } + inSlice[r.PathWithNamespace] = true } } @@ -385,93 +389,48 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } for k, gr := range gitlabgrouprepos { for _, r := range gr { - sub.Debug().Str("repo-type", "group").Msg(r.WebURL) - if repo.Filter.ExcludeForks { - if r.ForkedFromProject != nil { - continue + if !inSlice[r.PathWithNamespace] { + sub.Debug().Str("repo-type", "group").Msg(r.WebURL) + if repo.Filter.ExcludeForks { + if r.ForkedFromProject != nil { + continue + } } - } - if repo.Filter.ExcludeArchived { - if r.Archived { - continue + if repo.Filter.ExcludeArchived { + if r.Archived { + continue + } } - } - if len(repo.Filter.Languages) > 0 { - langs, _, err := client.Projects.GetProjectLanguages(r.ID) - if err != nil { - sub.Error(). - Msg(err.Error()) - continue - } else { - language := "" - percentage := float32(0) - - for lang, percent := range *langs { - if percent > percentage { - language = lang - } - } - if !languages[strings.ToLower(language)] { + if len(repo.Filter.Languages) > 0 { + langs, _, err := client.Projects.GetProjectLanguages(r.ID) + if err != nil { + sub.Error(). + Msg(err.Error()) continue + } else { + language := "" + percentage := float32(0) + + for lang, percent := range *langs { + if percent > percentage { + language = lang + } + } + if !languages[strings.ToLower(language)] { + continue + } } } - } - - if r.StarCount < repo.Filter.Stars { - continue - } - if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { - continue - } - if include[r.Name] { - if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { - repos = append(repos, types.Repo{ - Name: r.Path, - URL: r.HTTPURLToRepo, - SSHURL: r.SSHURLToRepo, - Token: token, - Defaultbranch: r.DefaultBranch, - Origin: repo, - Owner: k, - Hoster: types.GetHost(repo.URL), - Description: r.Description, - Private: r.Visibility == gitlab.PrivateVisibility, - Issues: GetIssues(r, client, repo), - }) + if r.StarCount < repo.Filter.Stars { + continue } - - if r.WikiEnabled && repo.Wiki { - if activeWiki(r, client, repo) { - httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") - sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") - repos = append(repos, types.Repo{ - Name: r.Path + ".wiki", - URL: httpURLToRepo, - SSHURL: sshURLToRepo, - Token: token, - Defaultbranch: r.DefaultBranch, - Origin: repo, - Owner: k, - Hoster: types.GetHost(repo.URL), - Description: r.Description, - Private: r.Visibility == gitlab.PrivateVisibility, - }) - } + if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { + continue } - continue - } - if exclude[r.Name] { - continue - } - if excludeorgs[r.Namespace.FullPath] { - continue - } - - if len(include) == 0 { - if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] { + if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ Name: r.Path, @@ -506,7 +465,55 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { }) } } + + continue + } + if exclude[r.Name] { + continue + } + if excludeorgs[r.Namespace.FullPath] { + continue + } + + if len(include) == 0 { + if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] { + if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { + repos = append(repos, types.Repo{ + Name: r.Path, + URL: r.HTTPURLToRepo, + SSHURL: r.SSHURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + Description: r.Description, + Private: r.Visibility == gitlab.PrivateVisibility, + Issues: GetIssues(r, client, repo), + }) + } + + if r.WikiEnabled && repo.Wiki { + if activeWiki(r, client, repo) { + httpURLToRepo := types.DotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git") + sshURLToRepo := types.DotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git") + repos = append(repos, types.Repo{ + Name: r.Path + ".wiki", + URL: httpURLToRepo, + SSHURL: sshURLToRepo, + Token: token, + Defaultbranch: r.DefaultBranch, + Origin: repo, + Owner: k, + Hoster: types.GetHost(repo.URL), + Description: r.Description, + Private: r.Visibility == gitlab.PrivateVisibility, + }) + } + } + } } + inSlice[r.PathWithNamespace] = true } } } From 345a5f0a3072971f2110546f429a7ac7d73aa119 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 28 Aug 2024 19:41:55 +0200 Subject: [PATCH 3/3] fix gitlab and better debug outputs --- gitea/gitea.go | 3 ++- gitlab/gitlab.go | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/gitea/gitea.go b/gitea/gitea.go index a90a3c5..c73e040 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -284,6 +284,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { languages := types.GetMap(repo.Filter.Languages) for _, r := range gitearepos { + sub.Debug().Str("repo-type", "user").Msg(r.HTMLURL) if repo.Filter.ExcludeForks { if r.Fork { continue @@ -296,7 +297,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } if len(repo.Filter.Languages) > 0 { - sub.Debug().Msg(r.HTMLURL) langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name) if err != nil { sub.Error(). @@ -430,6 +430,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } } for _, r := range orgrepos { + sub.Debug().Str("repo-type", "org").Msg(r.HTMLURL) if repo.Filter.ExcludeForks { if r.Fork { continue diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index 815d4b9..8bf6f00 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -122,6 +122,8 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + userNotInConfig := false + if repo.User == "" { user, _, err := client.Users.CurrentUser() if err != nil { @@ -130,6 +132,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } repo.User = user.Username + userNotInConfig = true } sub.Info(). @@ -151,7 +154,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { i := 1 for { opt.Page = i - projects, _, err := client.Projects.ListProjects(opt) + projects := []*gitlab.Project{} + if userNotInConfig { + projects, _, err = client.Projects.ListProjects(opt) + } else { + projects, _, err = client.Projects.ListUserProjects(user.ID, opt) + } if err != nil { sub.Error(). Msg(err.Error())