From f923cb378884b2cf9ae59f4cd62376f37dbae436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Tue, 12 Mar 2024 11:10:32 +0100 Subject: [PATCH 1/7] Handle action in the load function --- main.go | 60 ++++++++++++++++++++----------------------- template/userlist.tpl | 2 +- userlist/userlist.go | 7 ++++- 3 files changed, 35 insertions(+), 34 deletions(-) diff --git a/main.go b/main.go index 39af3c1..dc90d7e 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ package main import ( - config2 "github.com/prodyna/github-users/config" + config "github.com/prodyna/github-users/config" "github.com/prodyna/github-users/userlist" "log/slog" "os" @@ -22,42 +22,38 @@ type Config struct { } func main() { - config, err := config2.New() + c, err := config.New() if err != nil { slog.Error("Unable to create config", "error", err) os.Exit(1) } - switch config.Action { - case "userlist": - ulc := userlist.New( - userlist.WithEnterprise(config.Enterprise), - userlist.WithGithubToken(config.GithubToken), - userlist.WithTemplateFile(config.TemplateFile), - userlist.WithMarkdownFile(config.MarkdownFile), - ) - err := ulc.Validate() - if err != nil { - slog.Error("Invalid config", "error", err) - os.Exit(1) - } - err = ulc.Load() - if err != nil { - slog.Error("Unable to load userlist", "error", err) - os.Exit(1) - } - err = ulc.Print() - if err != nil { - slog.Error("Unable to print userlist", "error", err) - os.Exit(1) - } - err = ulc.Render() - if err != nil { - slog.Error("Unable to render userlist", "error", err) - os.Exit(1) - } - default: - slog.Error("Unknown action", "action", config.Action) + ulc := userlist.New( + userlist.WithAction(c.Action), + userlist.WithEnterprise(c.Enterprise), + userlist.WithGithubToken(c.GithubToken), + userlist.WithTemplateFile(c.TemplateFile), + userlist.WithMarkdownFile(c.MarkdownFile), + ) + + err = ulc.Validate() + if err != nil { + slog.Error("Invalid config", "error", err) + os.Exit(1) + } + err = ulc.Load() + if err != nil { + slog.Error("Unable to load userlist", "error", err) + os.Exit(1) + } + err = ulc.Print() + if err != nil { + slog.Error("Unable to print userlist", "error", err) + os.Exit(1) + } + err = ulc.Render() + if err != nil { + slog.Error("Unable to render userlist", "error", err) os.Exit(1) } } diff --git a/template/userlist.tpl b/template/userlist.tpl index 6920061..006bfaf 100644 --- a/template/userlist.tpl +++ b/template/userlist.tpl @@ -4,7 +4,7 @@ Last updated: {{ .Updated }} | # | GitHub Login | GitHub name | E-Mail | Contributions | | --- | --- | --- | --- | --- | -{{ range .Users }} | {{ .Number }} | [{{ .Login }}](https://github.com/enterprises/{{ $.Enterprise.Slug }}/people/{{ .Login }}/sso) | {{ .Name }} | {{ .Email }} | {{if .Contributions}}:green_square:{{else}}:red_square:{{end}} {{.Contributions }} | +{{ range .Users }} | {{ .Number }} | [{{ .Login }}](https://github.com/enterprises/{{ $.Enterprise.Slug }}/people/{{ .Login }}/sso) | {{ .Name }} | {{ .Email }} | {{if .Contributions}}:green_square:{{else}}:red_square:{{end}} [{{.Contributions }}](https://github.com/{{ .Name }}) | {{ end }} {{ if .Users }}_{{ len .Users }} users_{{ else }}No users found.{{ end }} diff --git a/userlist/userlist.go b/userlist/userlist.go index 531b5de..e9f82be 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -15,6 +15,7 @@ import ( ) type UserListConfig struct { + action string templateFile string markdownFile string enterprise string @@ -58,8 +59,9 @@ func New(options ...func(*UserListConfig)) *UserListConfig { return config } -func WithClient() func(*UserListConfig) { +func WithAction(action string) func(*UserListConfig) { return func(config *UserListConfig) { + config.action = action } } @@ -88,6 +90,9 @@ func WithMarkdownFile(markdownFile string) func(*UserListConfig) { } func (c *UserListConfig) Validate() error { + if c.action == "" { + return errors.New("Action is required") + } if c.templateFile == "" { return errors.New("Template is required") } From 5ee248cd07e0794da80201247e4ad1534cbc970e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Tue, 12 Mar 2024 11:24:53 +0100 Subject: [PATCH 2/7] Action support added --- config/config.go | 13 ++++++------- userlist/userlist.go | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/config/config.go b/config/config.go index 6b25981..cbf27aa 100644 --- a/config/config.go +++ b/config/config.go @@ -34,14 +34,13 @@ func New() (*Config, error) { flag.StringVar(&c.MarkdownFile, keyMarkdownFile, lookupEnvOrString("MARKDOWN_FILE", "USERS.md"), "The markdown file to write the result to.") verbose := flag.Int("verbose", lookupEnvOrInt(keyVerbose, 0), "Verbosity level, 0=info, 1=debug. Overrides the environment variable VERBOSE.") - logLevel := &slog.LevelVar{} - if verbose != nil { - if *verbose == 0 { - logLevel.Set(slog.LevelInfo) - } else { - logLevel.Set(slog.LevelDebug) - } + level := slog.LevelInfo + if *verbose > 0 { + level = slog.LevelDebug } + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: level, + }))) flag.Parse() return &c, nil } diff --git a/userlist/userlist.go b/userlist/userlist.go index e9f82be..b4e333c 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -14,6 +14,11 @@ import ( "time" ) +const ( + members = "members" + collaborators = "collaborators" +) + type UserListConfig struct { action string templateFile string @@ -107,6 +112,7 @@ func (c *UserListConfig) Validate() error { } c.validated = true slog.Debug("Validated userlist", + "action", c.action, "enterprise", c.enterprise, "template", c.templateFile, "githubToken", "***", @@ -118,7 +124,18 @@ func (c *UserListConfig) Load() error { if !c.validated { return errors.New("Config not validated") } - slog.Info("Loading userlist", "enterprise", c.enterprise) + switch c.action { + case members: + return c.loadMembers() + case collaborators: + return errors.New("Not implemented") + default: + return errors.New("Unknown action") + } +} + +func (c *UserListConfig) loadMembers() error { + slog.Info("Loading members", "enterprise", c.enterprise) c.userList = &UserList{ // updated as RFC3339 string Updated: time.Now().Format(time.RFC3339), @@ -172,6 +189,7 @@ func (c *UserListConfig) Load() error { } for offset := 0; ; offset += window { + slog.Debug("Running query", "offset", offset, "window", window) err := client.Query(ctx, &query, variables) if err != nil { slog.ErrorContext(ctx, "Unable to query", "error", err) From 222d6ddcf2e0d255d1f63effe5a7f7cd359843b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Tue, 12 Mar 2024 14:02:07 +0100 Subject: [PATCH 3/7] Loadig collaborators --- userlist/collaborators.go | 139 ++++++++++++++++++++++++++++++++++++++ userlist/constructor.go | 42 ++++++++++++ userlist/members.go | 98 +++++++++++++++++++++++++++ userlist/userlist.go | 139 +------------------------------------- 4 files changed, 282 insertions(+), 136 deletions(-) create mode 100644 userlist/collaborators.go create mode 100644 userlist/constructor.go create mode 100644 userlist/members.go diff --git a/userlist/collaborators.go b/userlist/collaborators.go new file mode 100644 index 0000000..c3e678f --- /dev/null +++ b/userlist/collaborators.go @@ -0,0 +1,139 @@ +package userlist + +import ( + "context" + "github.com/shurcooL/githubv4" + "golang.org/x/oauth2" + "log/slog" + "time" +) + +func (c *UserListConfig) loadCollaborators() error { + slog.Info("Loading collaborators", "enterprise", c.enterprise) + c.userList = &UserList{ + // updated as RFC3339 string + Updated: time.Now().Format(time.RFC3339), + } + ctx := context.Background() + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: c.githubToken}, + ) + httpClient := oauth2.NewClient(ctx, src) + client := githubv4.NewClient(httpClient) + + /* + { + enterprise(slug: "prodyna") { + slug + name + organizations (first:100) { + nodes { + login + name + } + } + } + } + */ + var organizations struct { + Enterprise struct { + Slug string + Name string + Organizations struct { + Nodes []struct { + Login string + Name string + } + } `graphql:"organizations(first:100)"` + } `graphql:"enterprise(slug: $slug)"` + } + + variables := map[string]interface{}{ + "slug": githubv4.String(c.enterprise), + } + + slog.Info("Loading organizations", "enterprise", c.enterprise) + err := client.Query(ctx, &organizations, variables) + if err != nil { + slog.ErrorContext(ctx, "Unable to query", "error", err) + return err + } + slog.Info("Loaded organizations", "organization.count", len(organizations.Enterprise.Organizations.Nodes)) + + /* + { + organization(login:"prodyna") { + repositories(first:100) { + pageInfo { + hasNextPage + startCursor + } + nodes { + name + collaborators(first:100,affiliation:OUTSIDE) { + pageInfo { + hasNextPage + startCursor + } + nodes { + login + name + } + } + } + } + } + } + */ + slog.Info("Iterating organizatons", "organization.count", len(organizations.Enterprise.Organizations.Nodes)) + for _, org := range organizations.Enterprise.Organizations.Nodes { + if org.Login != "PRODYNA" { + continue + } + slog.Info("Loading repositories and external collaborators", "organization", org.Login) + var repositories struct { + Organization struct { + Repositories struct { + Nodes []struct { + Name string + Collaborators struct { + Nodes []struct { + Login string + Name string + } + } `graphql:"collaborators(first:100,affiliation:OUTSIDE)"` + } + } `graphql:"repositories(first:100)"` + } `graphql:"organization(login: $organization)"` + } + + variables := map[string]interface{}{ + "organization": githubv4.String(org.Login), + } + + err := client.Query(ctx, &repositories, variables) + if err != nil { + slog.WarnContext(ctx, "Unable to query - will skip this organization", "error", err, "organization", org.Login) + continue + } + + // count the collaborators + collaboratorCount := 0 + for _, repo := range repositories.Organization.Repositories.Nodes { + collaboratorCount += len(repo.Collaborators.Nodes) + } + + slog.InfoContext(ctx, "Loaded repositories", + "repository.count", len(repositories.Organization.Repositories.Nodes), + "organization", org.Login, + "collaborator.count", collaboratorCount) + + if collaboratorCount == 0 { + continue + } + + slog.InfoContext(ctx, "Adding collaborators", "organization", org.Login) + } + + return nil +} diff --git a/userlist/constructor.go b/userlist/constructor.go new file mode 100644 index 0000000..3b4e5e3 --- /dev/null +++ b/userlist/constructor.go @@ -0,0 +1,42 @@ +package userlist + +func New(options ...func(*UserListConfig)) *UserListConfig { + config := &UserListConfig{ + validated: false, + loaded: false, + } + for _, option := range options { + option(config) + } + return config +} + +func WithAction(action string) func(*UserListConfig) { + return func(config *UserListConfig) { + config.action = action + } +} + +func WithTemplateFile(templateFile string) func(*UserListConfig) { + return func(config *UserListConfig) { + config.templateFile = templateFile + } +} + +func WithEnterprise(enterprise string) func(*UserListConfig) { + return func(config *UserListConfig) { + config.enterprise = enterprise + } +} + +func WithGithubToken(githubToken string) func(*UserListConfig) { + return func(config *UserListConfig) { + config.githubToken = githubToken + } +} + +func WithMarkdownFile(markdownFile string) func(*UserListConfig) { + return func(config *UserListConfig) { + config.markdownFile = markdownFile + } +} diff --git a/userlist/members.go b/userlist/members.go new file mode 100644 index 0000000..1c985d5 --- /dev/null +++ b/userlist/members.go @@ -0,0 +1,98 @@ +package userlist + +import ( + "context" + "github.com/shurcooL/githubv4" + "golang.org/x/oauth2" + "log/slog" + "time" +) + +func (c *UserListConfig) loadMembers() error { + slog.Info("Loading members", "enterprise", c.enterprise) + c.userList = &UserList{ + // updated as RFC3339 string + Updated: time.Now().Format(time.RFC3339), + } + + ctx := context.Background() + src := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: c.githubToken}, + ) + httpClient := oauth2.NewClient(ctx, src) + client := githubv4.NewClient(httpClient) + + var query struct { + Enterprise struct { + Slug string + Name string + OwnerInfo struct { + SamlIdentityProvider struct { + ExternalIdentities struct { + PageInfo struct { + HasNextPage bool + EndCursor githubv4.String + } + Edges []struct { + Node struct { + User struct { + Login string + Name string + ContributionsCollection struct { + ContributionCalendar struct { + TotalContributions int + } + } + } + SamlIdentity struct { + NameId string + } + } + } + } `graphql:"externalIdentities(after: $after, first: $first)"` + } + } + } `graphql:"enterprise(slug: $slug)"` + } + + window := 100 + variables := map[string]interface{}{ + "slug": githubv4.String("prodyna"), + "first": githubv4.Int(window), + "after": (*githubv4.String)(nil), + } + + for offset := 0; ; offset += window { + slog.Debug("Running query", "offset", offset, "window", window) + err := client.Query(ctx, &query, variables) + if err != nil { + slog.ErrorContext(ctx, "Unable to query", "error", err) + } + + c.userList.Enterprise = Enterprise{ + Slug: query.Enterprise.Slug, + Name: query.Enterprise.Name, + } + + for i, e := range query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.Edges { + u := User{ + Number: offset + i + 1, + Login: e.Node.User.Login, + Name: e.Node.User.Name, + Email: e.Node.SamlIdentity.NameId, + Contributions: e.Node.User.ContributionsCollection.ContributionCalendar.TotalContributions, + } + c.userList.Users = append(c.userList.Users, u) + } + + if !query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.PageInfo.HasNextPage { + break + } + + variables["after"] = githubv4.NewString(query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.PageInfo.EndCursor) + } + + slog.InfoContext(ctx, "Loaded userlist", "users", len(c.userList.Users)) + c.loaded = true + return nil +} diff --git a/userlist/userlist.go b/userlist/userlist.go index b4e333c..723fa9c 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -6,12 +6,9 @@ import ( "encoding/json" "errors" "fmt" - "github.com/shurcooL/githubv4" - "golang.org/x/oauth2" "log/slog" "os" "text/template" - "time" ) const ( @@ -25,9 +22,9 @@ type UserListConfig struct { markdownFile string enterprise string githubToken string - userList *UserList validated bool loaded bool + userList *UserList } type UserList struct { @@ -53,47 +50,6 @@ type Organization struct { Name string `json:"Name"` } -func New(options ...func(*UserListConfig)) *UserListConfig { - config := &UserListConfig{ - validated: false, - loaded: false, - } - for _, option := range options { - option(config) - } - return config -} - -func WithAction(action string) func(*UserListConfig) { - return func(config *UserListConfig) { - config.action = action - } -} - -func WithTemplateFile(templateFile string) func(*UserListConfig) { - return func(config *UserListConfig) { - config.templateFile = templateFile - } -} - -func WithEnterprise(enterprise string) func(*UserListConfig) { - return func(config *UserListConfig) { - config.enterprise = enterprise - } -} - -func WithGithubToken(githubToken string) func(*UserListConfig) { - return func(config *UserListConfig) { - config.githubToken = githubToken - } -} - -func WithMarkdownFile(markdownFile string) func(*UserListConfig) { - return func(config *UserListConfig) { - config.markdownFile = markdownFile - } -} - func (c *UserListConfig) Validate() error { if c.action == "" { return errors.New("Action is required") @@ -128,99 +84,10 @@ func (c *UserListConfig) Load() error { case members: return c.loadMembers() case collaborators: - return errors.New("Not implemented") + return c.loadCollaborators() default: - return errors.New("Unknown action") - } -} - -func (c *UserListConfig) loadMembers() error { - slog.Info("Loading members", "enterprise", c.enterprise) - c.userList = &UserList{ - // updated as RFC3339 string - Updated: time.Now().Format(time.RFC3339), - } - - ctx := context.Background() - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: c.githubToken}, - ) - httpClient := oauth2.NewClient(ctx, src) - client := githubv4.NewClient(httpClient) - - var query struct { - Enterprise struct { - Slug string - Name string - OwnerInfo struct { - SamlIdentityProvider struct { - ExternalIdentities struct { - PageInfo struct { - HasNextPage bool - EndCursor githubv4.String - } - Edges []struct { - Node struct { - User struct { - Login string - Name string - ContributionsCollection struct { - ContributionCalendar struct { - TotalContributions int - } - } - } - SamlIdentity struct { - NameId string - } - } - } - } `graphql:"externalIdentities(after: $after, first: $first)"` - } - } - } `graphql:"enterprise(slug: $slug)"` - } - - window := 100 - variables := map[string]interface{}{ - "slug": githubv4.String("prodyna"), - "first": githubv4.Int(window), - "after": (*githubv4.String)(nil), + return errors.New(fmt.Sprintf("Unknown action %s", c.action)) } - - for offset := 0; ; offset += window { - slog.Debug("Running query", "offset", offset, "window", window) - err := client.Query(ctx, &query, variables) - if err != nil { - slog.ErrorContext(ctx, "Unable to query", "error", err) - } - - c.userList.Enterprise = Enterprise{ - Slug: query.Enterprise.Slug, - Name: query.Enterprise.Name, - } - - for i, e := range query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.Edges { - u := User{ - Number: offset + i + 1, - Login: e.Node.User.Login, - Name: e.Node.User.Name, - Email: e.Node.SamlIdentity.NameId, - Contributions: e.Node.User.ContributionsCollection.ContributionCalendar.TotalContributions, - } - c.userList.Users = append(c.userList.Users, u) - } - - if !query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.PageInfo.HasNextPage { - break - } - - variables["after"] = githubv4.NewString(query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.PageInfo.EndCursor) - } - - slog.InfoContext(ctx, "Loaded userlist", "users", len(c.userList.Users)) - c.loaded = true - return nil } func (c *UserListConfig) Print() error { From 2f04c5dead568ae127ccc099198ed0b7f7a38978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Tue, 12 Mar 2024 20:34:49 +0100 Subject: [PATCH 4/7] Using upsert methods --- userlist/collaborators.go | 18 ++++++++++++++++-- userlist/members.go | 2 +- userlist/userlist.go | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/userlist/collaborators.go b/userlist/collaborators.go index c3e678f..09f0a2c 100644 --- a/userlist/collaborators.go +++ b/userlist/collaborators.go @@ -2,6 +2,8 @@ package userlist import ( "context" + "encoding/json" + "fmt" "github.com/shurcooL/githubv4" "golang.org/x/oauth2" "log/slog" @@ -98,8 +100,13 @@ func (c *UserListConfig) loadCollaborators() error { Name string Collaborators struct { Nodes []struct { - Login string - Name string + Login string + Name string + ContributionsCollection struct { + ContributionCalendar struct { + TotalContributions int + } + } } } `graphql:"collaborators(first:100,affiliation:OUTSIDE)"` } @@ -132,6 +139,13 @@ func (c *UserListConfig) loadCollaborators() error { continue } + output, err := json.MarshalIndent(repositories, "", " ") + if err != nil { + slog.ErrorContext(ctx, "Unable to marshal json", "error", err) + return err + } + fmt.Printf("%s\n", output) + slog.InfoContext(ctx, "Adding collaborators", "organization", org.Login) } diff --git a/userlist/members.go b/userlist/members.go index 1c985d5..7f0a8ed 100644 --- a/userlist/members.go +++ b/userlist/members.go @@ -82,7 +82,7 @@ func (c *UserListConfig) loadMembers() error { Email: e.Node.SamlIdentity.NameId, Contributions: e.Node.User.ContributionsCollection.ContributionCalendar.TotalContributions, } - c.userList.Users = append(c.userList.Users, u) + c.userList.upsertUser(u) } if !query.Enterprise.OwnerInfo.SamlIdentityProvider.ExternalIdentities.PageInfo.HasNextPage { diff --git a/userlist/userlist.go b/userlist/userlist.go index 723fa9c..b4cc712 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -44,9 +44,15 @@ type User struct { Name string `json:"Name"` Email string `json:"Email"` Contributions int `json:"Contributions"` + Organizations []Organization } type Organization struct { + Name string `json:"Name"` + Repositories []Repository `json:"Repositories"` +} + +type Repository struct { Name string `json:"Name"` } @@ -144,3 +150,33 @@ func (organization *Organization) RenderMarkdown(ctx context.Context, templateCo } return buffer.String(), nil } + +func (ul *UserList) upsertUser(user User) { + for i, u := range ul.Users { + if u.Login == user.Login { + ul.Users[i] = user + return + } + } + ul.Users = append(ul.Users, user) +} + +func (u *User) upsertOrganization(org Organization) { + for i, o := range u.Organizations { + if o.Name == org.Name { + u.Organizations[i] = org + return + } + } + u.Organizations = append(u.Organizations, org) +} + +func (o *Organization) upsertRepository(repo Repository) { + for i, r := range o.Repositories { + if r.Name == repo.Name { + o.Repositories[i] = repo + return + } + } + o.Repositories = append(o.Repositories, repo) +} From a68b3553de4ef3feb53a23db08495f98569f0de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Wed, 13 Mar 2024 00:14:29 +0100 Subject: [PATCH 5/7] Collaborators data now complete --- userlist/collaborators.go | 43 +++++++++++++++++++++++++++++++------ userlist/members.go | 2 +- userlist/userlist.go | 45 +++++++++++++++++++++------------------ 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/userlist/collaborators.go b/userlist/collaborators.go index 09f0a2c..8e44513 100644 --- a/userlist/collaborators.go +++ b/userlist/collaborators.go @@ -12,7 +12,7 @@ import ( func (c *UserListConfig) loadCollaborators() error { slog.Info("Loading collaborators", "enterprise", c.enterprise) - c.userList = &UserList{ + c.userList = UserList{ // updated as RFC3339 string Updated: time.Now().Format(time.RFC3339), } @@ -87,13 +87,16 @@ func (c *UserListConfig) loadCollaborators() error { } } */ + c.userList.Enterprise.Slug = organizations.Enterprise.Slug + c.userList.Enterprise.Name = organizations.Enterprise.Name + slog.Info("Iterating organizatons", "organization.count", len(organizations.Enterprise.Organizations.Nodes)) for _, org := range organizations.Enterprise.Organizations.Nodes { if org.Login != "PRODYNA" { continue } slog.Info("Loading repositories and external collaborators", "organization", org.Login) - var repositories struct { + var query struct { Organization struct { Repositories struct { Nodes []struct { @@ -118,7 +121,7 @@ func (c *UserListConfig) loadCollaborators() error { "organization": githubv4.String(org.Login), } - err := client.Query(ctx, &repositories, variables) + err := client.Query(ctx, &query, variables) if err != nil { slog.WarnContext(ctx, "Unable to query - will skip this organization", "error", err, "organization", org.Login) continue @@ -126,12 +129,39 @@ func (c *UserListConfig) loadCollaborators() error { // count the collaborators collaboratorCount := 0 - for _, repo := range repositories.Organization.Repositories.Nodes { + for _, repo := range query.Organization.Repositories.Nodes { collaboratorCount += len(repo.Collaborators.Nodes) } + if collaboratorCount == 0 { + slog.DebugContext(ctx, "No collaborators found", "organization", org.Login) + continue + } + + for _, repo := range query.Organization.Repositories.Nodes { + slog.DebugContext(ctx, "Processing repository", "repository", repo.Name, "collaborator.count", len(repo.Collaborators.Nodes)) + for _, collaborator := range repo.Collaborators.Nodes { + slog.DebugContext(ctx, "Processing collaborator", "login", collaborator.Login, "name", collaborator.Name, "contributions", collaborator.ContributionsCollection.ContributionCalendar.TotalContributions) + user := User{ + Login: collaborator.Login, + Name: collaborator.Name, + Organizations: new([]Organization), + Contributions: collaborator.ContributionsCollection.ContributionCalendar.TotalContributions, + } + c.userList.upsertUser(user) + organization := Organization{ + Name: repo.Name, + Repositories: new([]Repository), + } + user.upsertOrganization(organization) + repository := Repository{ + Name: repo.Name, + } + organization.upsertRepository(repository) + } + } slog.InfoContext(ctx, "Loaded repositories", - "repository.count", len(repositories.Organization.Repositories.Nodes), + "repository.count", len(query.Organization.Repositories.Nodes), "organization", org.Login, "collaborator.count", collaboratorCount) @@ -139,7 +169,7 @@ func (c *UserListConfig) loadCollaborators() error { continue } - output, err := json.MarshalIndent(repositories, "", " ") + output, err := json.MarshalIndent(c.userList, "", " ") if err != nil { slog.ErrorContext(ctx, "Unable to marshal json", "error", err) return err @@ -149,5 +179,6 @@ func (c *UserListConfig) loadCollaborators() error { slog.InfoContext(ctx, "Adding collaborators", "organization", org.Login) } + c.loaded = true return nil } diff --git a/userlist/members.go b/userlist/members.go index 7f0a8ed..6014db7 100644 --- a/userlist/members.go +++ b/userlist/members.go @@ -10,7 +10,7 @@ import ( func (c *UserListConfig) loadMembers() error { slog.Info("Loading members", "enterprise", c.enterprise) - c.userList = &UserList{ + c.userList = UserList{ // updated as RFC3339 string Updated: time.Now().Format(time.RFC3339), } diff --git a/userlist/userlist.go b/userlist/userlist.go index b4cc712..a9a9088 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -24,13 +24,13 @@ type UserListConfig struct { githubToken string validated bool loaded bool - userList *UserList + userList UserList } type UserList struct { Updated string Enterprise Enterprise - Users []User + Users []*User } type Enterprise struct { @@ -44,12 +44,12 @@ type User struct { Name string `json:"Name"` Email string `json:"Email"` Contributions int `json:"Contributions"` - Organizations []Organization + Organizations *[]Organization } type Organization struct { - Name string `json:"Name"` - Repositories []Repository `json:"Repositories"` + Name string `json:"Name"` + Repositories *[]Repository `json:"Repositories"` } type Repository struct { @@ -154,29 +154,32 @@ func (organization *Organization) RenderMarkdown(ctx context.Context, templateCo func (ul *UserList) upsertUser(user User) { for i, u := range ul.Users { if u.Login == user.Login { - ul.Users[i] = user + *ul.Users[i] = user return } } - ul.Users = append(ul.Users, user) + slog.Info("Upserting user", "login", user.Login) + ul.Users = append(ul.Users, &user) } func (u *User) upsertOrganization(org Organization) { - for i, o := range u.Organizations { - if o.Name == org.Name { - u.Organizations[i] = org - return - } - } - u.Organizations = append(u.Organizations, org) + //for i, o := range u.Organizations { + // if o.Name == org.Name { + // u.Organizations[i] = org + // return + // } + //} + *u.Organizations = append(*u.Organizations, org) + slog.Debug("Upserting organization", "name", org.Name, "user", u.Login) } func (o *Organization) upsertRepository(repo Repository) { - for i, r := range o.Repositories { - if r.Name == repo.Name { - o.Repositories[i] = repo - return - } - } - o.Repositories = append(o.Repositories, repo) + //for i, r := range o.Repositories { + // if r.Name == repo.Name { + // o.Repositories[i] = repo + // return + // } + //} + slog.Debug("Upserting repository", "name", repo.Name, "organization", o.Name) + *o.Repositories = append(*o.Repositories, repo) } From 2c6b57198d947f0c6e19f5082d5a78006ff6fbb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Wed, 13 Mar 2024 12:10:54 +0100 Subject: [PATCH 6/7] Current state, members and collaborators --- README.md | 27 +++++++++++++++++++++----- action.yaml | 4 ++-- config/config.go | 2 +- template/collaborators.tpl | 11 +++++++++++ template/{userlist.tpl => members.tpl} | 2 +- userlist/collaborators.go | 5 ++++- 6 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 template/collaborators.tpl rename template/{userlist.tpl => members.tpl} (91%) diff --git a/README.md b/README.md index 12c54fe..b9ecc1e 100644 --- a/README.md +++ b/README.md @@ -41,18 +41,35 @@ jobs: # Run the deployment overview action - name: Github users - uses: prodyna/github-users@v0.7 + uses: prodyna/github-users@v1.0 with: # The action to run - action: userlist + action: members # The GitHub Enterprise to query for repositories enterprise: octocat # The GitHub Token to use for authentication github-token: ${{ secrets.GITHUB_TOKEN }} # The template file to use for rendering the result - template-file: template/userlist.tpl + template-file: template/members.tpl # The markdown file to write the result to - markdown-file: USERS.md + markdown-file: MEMBERS.md + # Verbosity level, 0=info, 1=debug + verbose: 1 + + # Run the deployment overview action + - name: Github users + uses: prodyna/github-users@v1.0 + with: + # The action to run + action: collaborators + # The GitHub Enterprise to query for repositories + enterprise: octocat + # The GitHub Token to use for authentication + github-token: ${{ secrets.GITHUB_TOKEN }} + # The template file to use for rendering the result + template-file: template/collaborators.tpl + # The markdown file to write the result to + markdown-file: COLLABORATORS.md # Verbosity level, 0=info, 1=debug verbose: 1 @@ -61,6 +78,6 @@ jobs: run: | git config --local user.email "darko@krizic.net" git config --local user.name "Deployment Overview" - git add profile + git add MEMBERS.md COLLABORATORS.md git commit -m "Add/update deployment overview" ``` diff --git a/action.yaml b/action.yaml index c5ba157..393b810 100644 --- a/action.yaml +++ b/action.yaml @@ -14,7 +14,7 @@ inputs: template-file: description: 'The template file to use for rendering the result' required: false - default: '/template/userlist.tpl' + default: '/template/members.tpl' markdown-file: description: 'The markdown file to write the result to' required: false @@ -25,7 +25,7 @@ inputs: default: 1 runs: using: 'docker' - image: 'docker://ghcr.io/prodyna/github-users:v0.7' + image: 'docker://ghcr.io/prodyna/github-users:v1.0' env: ACTION: ${{ inputs.action }} ENTERPRISE: ${{ inputs.enterprise }} diff --git a/config/config.go b/config/config.go index cbf27aa..3592313 100644 --- a/config/config.go +++ b/config/config.go @@ -30,7 +30,7 @@ func New() (*Config, error) { flag.StringVar(&c.Action, keyAction, lookupEnvOrString("ACTION", ""), "The action to perform.") flag.StringVar(&c.Enterprise, keyEnterprise, lookupEnvOrString("ENTERPRISE", ""), "The GitHub Enterprise to query for repositories.") flag.StringVar(&c.GithubToken, keyGithubToken, lookupEnvOrString("GITHUB_TOKEN", ""), "The GitHub Token to use for authentication.") - flag.StringVar(&c.TemplateFile, keyTemplateFile, lookupEnvOrString("TEMPLATE_FILE", "template/userlist.tpl"), "The template file to use for rendering the result.") + flag.StringVar(&c.TemplateFile, keyTemplateFile, lookupEnvOrString("TEMPLATE_FILE", "template/members.tpl"), "The template file to use for rendering the result.") flag.StringVar(&c.MarkdownFile, keyMarkdownFile, lookupEnvOrString("MARKDOWN_FILE", "USERS.md"), "The markdown file to write the result to.") verbose := flag.Int("verbose", lookupEnvOrInt(keyVerbose, 0), "Verbosity level, 0=info, 1=debug. Overrides the environment variable VERBOSE.") diff --git a/template/collaborators.tpl b/template/collaborators.tpl new file mode 100644 index 0000000..9aec363 --- /dev/null +++ b/template/collaborators.tpl @@ -0,0 +1,11 @@ +# GitHub Enterprise collaborators for {{ .Enterprise.Name }} + +Last updated: {{ .Updated }} + +| Number | User | Contributions | Organization | Repository | +| ------ | ---- | ------------- | ------------ | ---------- | +{{ range $user := .Users }}{{ range $org := $user.Organizations }}{{ range $repo := $org.Repositories }}| {{ $user.Number }} | {{ $user.Login }} | {{ $user.Contributions }} | {{ $org.Name }} | {{ $repo.Name }} | +{{ end }}{{ end }}{{ end }} + +--- +Generated with :heart: by [github-users](https://github.com/prodyna/github-users) diff --git a/template/userlist.tpl b/template/members.tpl similarity index 91% rename from template/userlist.tpl rename to template/members.tpl index 006bfaf..8e30774 100644 --- a/template/userlist.tpl +++ b/template/members.tpl @@ -1,4 +1,4 @@ -# GitHub Enterprise Users for {{ .Enterprise.Name }} +# GitHub Enterprise members for {{ .Enterprise.Name }} Last updated: {{ .Updated }} diff --git a/userlist/collaborators.go b/userlist/collaborators.go index 8e44513..293246d 100644 --- a/userlist/collaborators.go +++ b/userlist/collaborators.go @@ -90,6 +90,7 @@ func (c *UserListConfig) loadCollaborators() error { c.userList.Enterprise.Slug = organizations.Enterprise.Slug c.userList.Enterprise.Name = organizations.Enterprise.Name + userNumber := 0 slog.Info("Iterating organizatons", "organization.count", len(organizations.Enterprise.Organizations.Nodes)) for _, org := range organizations.Enterprise.Organizations.Nodes { if org.Login != "PRODYNA" { @@ -142,14 +143,16 @@ func (c *UserListConfig) loadCollaborators() error { for _, collaborator := range repo.Collaborators.Nodes { slog.DebugContext(ctx, "Processing collaborator", "login", collaborator.Login, "name", collaborator.Name, "contributions", collaborator.ContributionsCollection.ContributionCalendar.TotalContributions) user := User{ + Number: userNumber + 1, Login: collaborator.Login, Name: collaborator.Name, Organizations: new([]Organization), Contributions: collaborator.ContributionsCollection.ContributionCalendar.TotalContributions, } c.userList.upsertUser(user) + userNumber++ organization := Organization{ - Name: repo.Name, + Name: org.Name, Repositories: new([]Repository), } user.upsertOrganization(organization) From eedfb22fae59b668a7422ae3d35701abdfb8ecbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Kri=C5=BEi=C4=87?= Date: Wed, 13 Mar 2024 12:38:16 +0100 Subject: [PATCH 7/7] upserting works now --- userlist/collaborators.go | 14 +++++------ userlist/userlist.go | 50 +++++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/userlist/collaborators.go b/userlist/collaborators.go index 293246d..5cf479f 100644 --- a/userlist/collaborators.go +++ b/userlist/collaborators.go @@ -142,15 +142,13 @@ func (c *UserListConfig) loadCollaborators() error { slog.DebugContext(ctx, "Processing repository", "repository", repo.Name, "collaborator.count", len(repo.Collaborators.Nodes)) for _, collaborator := range repo.Collaborators.Nodes { slog.DebugContext(ctx, "Processing collaborator", "login", collaborator.Login, "name", collaborator.Name, "contributions", collaborator.ContributionsCollection.ContributionCalendar.TotalContributions) - user := User{ - Number: userNumber + 1, - Login: collaborator.Login, - Name: collaborator.Name, - Organizations: new([]Organization), - Contributions: collaborator.ContributionsCollection.ContributionCalendar.TotalContributions, + user := c.userList.findUser(collaborator.Login) + if user == nil { + user = c.userList.createUser(userNumber+1, collaborator.Login, collaborator.Name, "", collaborator.ContributionsCollection.ContributionCalendar.TotalContributions) + userNumber++ + } else { + slog.Info("Found existing user", "login", user.Login) } - c.userList.upsertUser(user) - userNumber++ organization := Organization{ Name: org.Name, Repositories: new([]Repository), diff --git a/userlist/userlist.go b/userlist/userlist.go index a9a9088..927b88e 100644 --- a/userlist/userlist.go +++ b/userlist/userlist.go @@ -162,24 +162,48 @@ func (ul *UserList) upsertUser(user User) { ul.Users = append(ul.Users, &user) } +func (ul *UserList) findUser(login string) *User { + for _, u := range ul.Users { + if u.Login == login { + return u + } + } + return nil +} + +func (ul *UserList) createUser(number int, login string, name string, email string, contributions int) *User { + user := &User{ + Number: number, + Login: login, + Name: name, + Email: email, + Contributions: contributions, + Organizations: new([]Organization), + } + ul.upsertUser(*user) + return user +} + func (u *User) upsertOrganization(org Organization) { - //for i, o := range u.Organizations { - // if o.Name == org.Name { - // u.Organizations[i] = org - // return - // } - //} + for _, o := range *u.Organizations { + if o.Name == org.Name { + // organization was found + for _, repo := range *org.Repositories { + o.upsertRepository(repo) + } + return + } + } *u.Organizations = append(*u.Organizations, org) - slog.Debug("Upserting organization", "name", org.Name, "user", u.Login) } func (o *Organization) upsertRepository(repo Repository) { - //for i, r := range o.Repositories { - // if r.Name == repo.Name { - // o.Repositories[i] = repo - // return - // } - //} + for _, r := range *o.Repositories { + if r.Name == repo.Name { + // repo was found + return + } + } slog.Debug("Upserting repository", "name", repo.Name, "organization", o.Name) *o.Repositories = append(*o.Repositories, repo) }