From 9cc642877177a1f5319271714cde6edefa71fcb8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Sun, 29 Oct 2023 21:40:53 -0700 Subject: [PATCH] Add info from Github pull requests and security advisories Add highlights to template and option to use highlights Signed-off-by: Derek McGowan --- github.go | 200 ++++++++++++++++++++++++++++++++++++++++++++-------- main.go | 55 +++++++++++---- template.go | 15 ++++ util.go | 63 +++++++++++++++++ 4 files changed, 290 insertions(+), 43 deletions(-) diff --git a/github.go b/github.go index 69da71c..a3e0bde 100644 --- a/github.go +++ b/github.go @@ -31,14 +31,18 @@ import ( var prr = regexp.MustCompile(`^Merge pull request(?: #([0-9]+))? from (\S+)$`) type githubChangeProcessor struct { - repo string - cache Cache // Need a way to expire or bypass cache + repo string + linkName string + cache Cache + refreshCache bool } -func githubChange(repo string, cache Cache) changeProcessor { +func githubChange(repo, linkName string, cache Cache, refreshCache bool) changeProcessor { return &githubChangeProcessor{ - repo: repo, - cache: cache, + repo: repo, + linkName: linkName, + cache: cache, + refreshCache: refreshCache, } } @@ -50,17 +54,20 @@ func (p *githubChangeProcessor) process(c *change) error { return err } - title, err := getPRTitle(p.repo, pr, p.cache) + info, err := p.getPRInfo(p.repo, pr) if err != nil { return err } + p.prChange(c, info, pr) - c.Title = title - c.Link = fmt.Sprintf("https://github.com/%s/pull/%d", p.repo, pr) - c.Formatted = fmt.Sprintf("%s ([#%d](%s))", c.Title, pr, c.Link) } else if strings.HasPrefix(string(matches[2]), "GHSA-") { - c.Link = fmt.Sprintf("https://github.com/%s/security/advisories/%s", p.repo, matches[2]) - c.Formatted = fmt.Sprintf("Github Security Advisory [%s](%s)", matches[2], c.Link) + ghsa := string(matches[2]) + info, err := p.getAdvisoryInfo(p.repo, ghsa) + if err != nil { + return err + } + p.advisoryChange(c, info, ghsa) + } else { logrus.Debugf("Nothing matched: %q", c.Description) } @@ -83,26 +90,73 @@ func (p *githubChangeProcessor) process(c *change) error { return nil } -// getPRTitle returns the Pull Request title from the github API -// TODO: Update to also return labels -func getPRTitle(repo string, prn int64, cache Cache) (string, error) { +func (p *githubChangeProcessor) prChange(c *change, info pullRequestInfo, pr int64) { + for _, l := range info.Labels { + if l.Name == "impact/changelog" { + c.IsHighlight = true + } else if l.Name == "impact/breaking" { + c.IsBreaking = true + } else if l.Name == "impact/deprecation" { + c.IsDeprecation = true + } else if strings.HasPrefix(l.Name, "area/") { + if l.Description != "" { + c.Category = l.Description + } else { + c.Category = l.Name[5:] + } + } + } + c.Title = info.Title + if len(c.Title) > 0 && c.Title[0] == '[' { + idx := strings.IndexByte(c.Title, ']') + if idx > 0 { + c.Title = strings.TrimSpace(c.Title[idx:]) + } + } + + if c.Link == "" { + c.Link = fmt.Sprintf("https://github.com/%s/pull/%d", p.repo, pr) + } + c.Formatted = fmt.Sprintf("%s ([%s#%d](%s))", c.Title, p.linkName, pr, c.Link) +} + +type pullRequestLabel struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type pullRequestInfo struct { + Title string `json:"title"` + Labels []pullRequestLabel `json:"labels"` +} + +// getPRInfo returns the Pull Request info from the github API +// +// See https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#get-a-pull-request +func (p *githubChangeProcessor) getPRInfo(repo string, prn int64) (pullRequestInfo, error) { u := fmt.Sprintf("https://api.github.com/repos/%s/pulls/%d", repo, prn) - key := u + " title" - if b, ok := cache.Get(key); ok { // TODO: Provide option to refresh cache - return string(b), nil + key := u + " title labels" + if !p.refreshCache { + if b, ok := p.cache.Get(key); ok { + var info pullRequestInfo + if err := json.Unmarshal(b, &info); err == nil { + return info, nil + } + } } req, err := http.NewRequest("GET", u, nil) if err != nil { - return "", err + return pullRequestInfo{}, err } - req.Header.Add("Accept", "application/vnd.github.v3+json") + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") if user, token := os.Getenv("GITHUB_ACTOR"), os.Getenv("GITHUB_TOKEN"); user != "" && token != "" { req.SetBasicAuth(user, token) } resp, err := http.DefaultClient.Do(req) if err != nil { - return "", err + return pullRequestInfo{}, err } defer resp.Body.Close() @@ -110,21 +164,107 @@ func getPRTitle(repo string, prn int64, cache Cache) (string, error) { if resp.StatusCode >= 403 { logrus.Warn("Forbidden response, try setting GITHUB_USER and GITHUB_TOKEN environment variables") } - return "", fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, u) + return pullRequestInfo{}, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, u) } dec := json.NewDecoder(resp.Body) - pr := struct { - Title string `json:"title"` - }{} - if err := dec.Decode(&pr); err != nil { - return "", err + var info pullRequestInfo + if err := dec.Decode(&info); err != nil { + return pullRequestInfo{}, err + } + if info.Title == "" { + return pullRequestInfo{}, fmt.Errorf("unexpected empty title for %s", u) + } + + cacheB, err := json.Marshal(info) + if err == nil { + p.cache.Put(key, cacheB) + } + + return info, nil +} + +func (p *githubChangeProcessor) advisoryChange(c *change, info advisoryInfo, ghsa string) { + c.IsSecurity = true + c.Link = info.Link + if c.Link == "" { + c.Link = fmt.Sprintf("https://github.com/%s/security/advisories/%s", p.repo, ghsa) + } + summary := info.Summary + if summary == "" { + summary = "Github Security Advisory" + } + c.Formatted = fmt.Sprintf("%s [%s](%s)", summary, ghsa, c.Link) + cveInfo := []string{} + if info.CVE != "" { + cveInfo = append(cveInfo, info.CVE) + } + if info.Severity != "" { + cveInfo = append(cveInfo, info.Severity) + } + if len(cveInfo) > 0 { + prefix := "[" + strings.Join(cveInfo, ", ") + "] " + c.Formatted = prefix + c.Formatted } - if pr.Title == "" { - return "", fmt.Errorf("unexpected empty title for %s", u) +} + +type advisoryInfo struct { + CVE string `json:"cve_id"` + Link string `json:"html_url"` + Summary string `json:"summary"` + Description string `json:"description"` + Severity string `json:"severity"` +} + +// getAdvisoryInfo returns github security advisory info +// +// See https://docs.github.com/en/rest/security-advisories/repository-advisories?apiVersion=2022-11-28#get-a-repository-security-advisory +func (p *githubChangeProcessor) getAdvisoryInfo(repo, advisory string) (advisoryInfo, error) { + u := fmt.Sprintf("https://api.github.com/repos/%s/security-advisories/%s", repo, advisory) + key := u + " cve link summary description severity" + if !p.refreshCache { + if b, ok := p.cache.Get(key); ok { + var info advisoryInfo + if err := json.Unmarshal(b, &info); err == nil { + return info, nil + } + } + } + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return advisoryInfo{}, err + } + req.Header.Add("Accept", "application/vnd.github+json") + req.Header.Add("X-GitHub-Api-Version", "2022-11-28") + if user, token := os.Getenv("GITHUB_ACTOR"), os.Getenv("GITHUB_TOKEN"); user != "" && token != "" { + req.SetBasicAuth(user, token) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return advisoryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + if resp.StatusCode >= 403 { + logrus.Warn("Forbidden response, try setting GITHUB_USER and GITHUB_TOKEN environment variables") + } + return advisoryInfo{}, fmt.Errorf("unexpected status code %d for %s", resp.StatusCode, u) + } + + dec := json.NewDecoder(resp.Body) + + var info advisoryInfo + if err := dec.Decode(&info); err != nil { + return advisoryInfo{}, err + } + + cacheB, err := json.Marshal(info) + if err == nil { + p.cache.Put(key, cacheB) } - cache.Put(key, []byte(pr.Title)) - return pr.Title, nil + return info, nil } diff --git a/main.go b/main.go index b9cbf8f..cf2ebfa 100644 --- a/main.go +++ b/main.go @@ -45,9 +45,11 @@ type change struct { Category string Link string - IsMerge bool - //IsBreaking bool - //IsHighlight bool + IsMerge bool + IsHighlight bool + IsBreaking bool + IsDeprecation bool + IsSecurity bool Formatted string } @@ -90,6 +92,16 @@ type contributor struct { OtherNames []string } +type highlightChange struct { + Project string + Change *change +} + +type highlightCategory struct { + Name string + Changes []highlightChange +} + type release struct { ProjectName string `toml:"project_name"` GithubRepo string `toml:"github_repo"` @@ -120,6 +132,7 @@ type release struct { // generated fields Changes []projectChange + Highlights []highlightCategory Contributors []contributor Dependencies []dependency Tag string @@ -160,6 +173,11 @@ This tool should be ran from the root of the project repository for a new releas Aliases: []string{"l"}, Usage: "add links to changelog", }, + &cli.BoolFlag{ + Name: "highlights", + Aliases: []string{"g"}, + Usage: "use highlights based on pull request", + }, &cli.BoolFlag{ Name: "short", Aliases: []string{"s"}, @@ -174,14 +192,21 @@ This tool should be ran from the root of the project repository for a new releas Usage: "cache directory for static remote resources", EnvVars: []string{"RELEASE_TOOL_CACHE"}, }, + &cli.BoolFlag{ + Name: "refresh-cache", + Aliases: []string{"r"}, + Usage: "refreshes cache", + }, } app.Action = func(context *cli.Context) error { var ( - releasePath = context.Args().First() - tag = context.String("tag") - linkify = context.Bool("linkify") - short = context.Bool("short") - skipCommits = context.Bool("skip-commits") + releasePath = context.Args().First() + tag = context.String("tag") + linkify = context.Bool("linkify") + highlights = context.Bool("highlights") + short = context.Bool("short") + skipCommits = context.Bool("skip-commits") + refreshCache = context.Bool("refresh-cache") ) if tag == "" { tag = parseTag(releasePath) @@ -237,9 +262,9 @@ This tool should be ran from the root of the project repository for a new releas if err != nil { return err } - if linkify { + if linkify || highlights { for _, change := range changes { - if err := githubChange(r.GithubRepo, cache).process(change); err != nil { + if err := githubChange(r.GithubRepo, "", cache, refreshCache).process(change); err != nil { return err } if !change.IsMerge { @@ -350,13 +375,13 @@ This tool should be ran from the root of the project repository for a new releas if err := addContributors(dep.Previous, dep.Ref, contributors); err != nil { return fmt.Errorf("failed to get authors for %s: %w", name, err) } - if linkify { + if linkify || highlights { if !strings.HasPrefix(dep.Name, "github.com/") { logrus.Debugf("linkify only supported for Github, skipping %s", dep.Name) } else { ghname := dep.Name[11:] for _, change := range changes { - if err := githubChange(ghname, cache).process(change); err != nil { + if err := githubChange(ghname, ghname, cache, refreshCache).process(change); err != nil { return err } if !change.IsMerge { @@ -388,7 +413,11 @@ This tool should be ran from the root of the project repository for a new releas // update the release fields with generated data r.Contributors = orderContributors(contributors) r.Dependencies = updatedDeps - r.Changes = projectChanges + if highlights { + r.Highlights = groupHighlights(projectChanges) + } else { + r.Changes = projectChanges + } r.Tag = tag r.Version = version diff --git a/template.go b/template.go index 238b780..5262825 100644 --- a/template.go +++ b/template.go @@ -27,6 +27,21 @@ Welcome to the {{.Tag}} release of {{.ProjectName}}! {{.Preface}} +{{- if .Highlights}} + +### Highlights +{{- range $highlight := .Highlights}} + +{{- if $highlight.Name}} + +#### {{$highlight.Name}} +{{- end}} +{{ range $change := $highlight.Changes}} +* {{ $change.Change.Formatted }} +{{- end}} +{{- end}} +{{- end}} + Please try out the release binaries and report any issues at https://github.com/{{.GithubRepo}}/issues. diff --git a/util.go b/util.go index a8363d0..53500b5 100644 --- a/util.go +++ b/util.go @@ -654,6 +654,69 @@ func orderContributors(contributors map[string]contributor) []contributor { return all } +func groupHighlights(changes []projectChange) []highlightCategory { + security := []highlightChange{} + deprecation := []highlightChange{} + breaking := []highlightChange{} + categories := map[string][]highlightChange{} + categoryList := []string{} + for _, project := range changes { + for _, c := range project.Changes { + if c.IsSecurity { + security = append(security, getHighlightChange(project.Name, c)) + } else if c.IsHighlight { + cc, ok := categories[c.Category] + if !ok { + categoryList = append(categoryList, c.Category) + } + categories[c.Category] = append(cc, getHighlightChange(project.Name, c)) + } + + // Allow deprecation and breaking changes to show up twice + if c.IsDeprecation { + deprecation = append(deprecation, getHighlightChange(project.Name, c)) + } else if c.IsBreaking { + breaking = append(breaking, getHighlightChange(project.Name, c)) + } + } + } + highlights := make([]highlightCategory, 0, len(categories)+3) + sort.Strings(categoryList) + for _, category := range categoryList { + highlights = append(highlights, highlightCategory{ + Name: category, + Changes: categories[category], + }) + } + if len(security) > 0 { + highlights = append(highlights, highlightCategory{ + Name: "Security Advisories", + Changes: security, + }) + } + if len(breaking) > 0 { + highlights = append(highlights, highlightCategory{ + Name: "Breaking", + Changes: breaking, + }) + } + if len(deprecation) > 0 { + highlights = append(highlights, highlightCategory{ + Name: "Deprecations", + Changes: deprecation, + }) + } + + return highlights +} + +func getHighlightChange(project string, c *change) highlightChange { + return highlightChange{ + Project: project, + Change: c, + } +} + // getTemplate will use a builtin template if the template is not specified on the cli func getTemplate(context *cli.Context) (string, error) { path := context.String("template")