From 6de5bcc25a049075ec40d8a329daff27977fa89e Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 27 Feb 2024 13:36:47 -0600 Subject: [PATCH 1/4] feat: initial start of github app --- cmd/vela-server/scm.go | 2 ++ scm/flags.go | 12 ++++++++++++ scm/github/github.go | 3 +++ scm/github/opts.go | 24 ++++++++++++++++++++++++ scm/setup.go | 5 +++++ 5 files changed, 46 insertions(+) diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index 3a7e0ef73..d324bdb18 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -24,6 +24,8 @@ func setupSCM(c *cli.Context) (scm.Service, error) { StatusContext: c.String("scm.context"), WebUIAddress: c.String("webui-addr"), Scopes: c.StringSlice("scm.scopes"), + GithubAppID: c.Int64("scm.app.id"), + GithubAppPrivateKey: c.String("scm.app.private_key"), } // setup the scm diff --git a/scm/flags.go b/scm/flags.go index 84a9e879c..2f27f1112 100644 --- a/scm/flags.go +++ b/scm/flags.go @@ -66,4 +66,16 @@ var Flags = []cli.Flag{ "is behind a Firewall or NAT, or when using something like ngrok to forward webhooks. " + "(defaults to VELA_ADDR).", }, + &cli.Int64Flag{ + EnvVars: []string{"VELA_SCM_APP_ID", "SCM_APP_ID"}, + FilePath: "/vela/scm/app_id", + Name: "scm.app.id", + Usage: "(optional & experimental) ID for the GitHub App", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY", "SCM_APP_PRIVATE_KEY"}, + FilePath: "/vela/scm/app_private_key", + Name: "scm.app.private_key", + Usage: "(optional & experimental) private key for the GitHub App", + }, } diff --git a/scm/github/github.go b/scm/github/github.go index d588f1795..2ca1e8880 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -45,6 +45,9 @@ type config struct { WebUIAddress string // specifies the OAuth scopes to use for the GitHub client Scopes []string + // optional and experimental + GithubAppID int64 + GithubAppPrivateKey string } type client struct { diff --git a/scm/github/opts.go b/scm/github/opts.go index df6d46506..4623ce279 100644 --- a/scm/github/opts.go +++ b/scm/github/opts.go @@ -149,3 +149,27 @@ func WithScopes(scopes []string) ClientOpt { return nil } } + +// WithGithubAppID sets the ID for the GitHub App in the scm client. +func WithGithubAppID(id int64) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring ID for GitHub App in github scm client") + + // set the ID for the GitHub App in the github client + c.config.GithubAppID = id + + return nil + } +} + +// WithGithubPrivateKey sets the private key for the GitHub App in the scm client. +func WithGithubPrivateKey(key string) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring private key for GitHub App in github scm client") + + // set the private key for the GitHub App in the github client + c.config.GithubAppPrivateKey = key + + return nil + } +} diff --git a/scm/setup.go b/scm/setup.go index b779e8ab2..92c5d0324 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -36,6 +36,9 @@ type Setup struct { WebUIAddress string // specifies the OAuth scopes to use for the scm client Scopes []string + // optional and experimental + GithubAppID int64 + GithubAppPrivateKey string } // Github creates and returns a Vela service capable of @@ -55,6 +58,8 @@ func (s *Setup) Github() (Service, error) { github.WithStatusContext(s.StatusContext), github.WithWebUIAddress(s.WebUIAddress), github.WithScopes(s.Scopes), + github.WithGithubAppID(s.GithubAppID), + github.WithGithubPrivateKey(s.GithubAppPrivateKey), ) } From 7b89f5bd093521bd97d980ecbc875a4f6d5a3762 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 27 Feb 2024 14:18:26 -0600 Subject: [PATCH 2/4] feat: create token from github app --- go.mod | 3 +++ go.sum | 6 +++++ scm/flags.go | 2 +- scm/github/github.go | 60 +++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 3cfaeb5fb..4eba76d21 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/adhocore/gronx v1.6.7 github.com/alicebob/miniredis/v2 v2.31.1 github.com/aws/aws-sdk-go v1.50.24 + github.com/bradleyfalzon/ghinstallation/v2 v2.9.0 github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/gin-gonic/gin v1.9.1 @@ -66,8 +67,10 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/gomodule/redigo v2.0.0+incompatible // indirect + github.com/google/go-github/v57 v57.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/css v1.0.0 // indirect diff --git a/go.sum b/go.sum index d2dde2a27..38a2dc84a 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd3 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bradleyfalzon/ghinstallation/v2 v2.9.0 h1:HmxIYqnxubRYcYGRc5v3wUekmo5Wv2uX3gukmWJ0AFk= +github.com/bradleyfalzon/ghinstallation/v2 v2.9.0/go.mod h1:wmkTDJf8CmVypxE8ijIStFnKoTa6solK5QfdmJrP9KI= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -92,6 +94,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -108,6 +112,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs= +github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw= github.com/google/go-github/v59 v59.0.0 h1:7h6bgpF5as0YQLLkEiVqpgtJqjimMYhBkD4jT5aN3VA= github.com/google/go-github/v59 v59.0.0/go.mod h1:rJU4R0rQHFVFDOkqGWxfLNo6vEk4dv40oDjhV/gH6wM= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= diff --git a/scm/flags.go b/scm/flags.go index 2f27f1112..45cb2d055 100644 --- a/scm/flags.go +++ b/scm/flags.go @@ -76,6 +76,6 @@ var Flags = []cli.Flag{ EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY", "SCM_APP_PRIVATE_KEY"}, FilePath: "/vela/scm/app_private_key", Name: "scm.app.private_key", - Usage: "(optional & experimental) private key for the GitHub App", + Usage: "(optional & experimental) path to private key for the GitHub App", }, } diff --git a/scm/github/github.go b/scm/github/github.go index 2ca1e8880..ac17af4f4 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -5,8 +5,13 @@ package github import ( "context" "fmt" + "net/http" "net/url" + "strings" + "github.com/go-vela/types/library" + + "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v59/github" "github.com/sirupsen/logrus" @@ -51,9 +56,10 @@ type config struct { } type client struct { - config *config - OAuth *oauth2.Config - AuthReq *github.AuthorizationRequest + config *config + OAuth *oauth2.Config + AuthReq *github.AuthorizationRequest + AppsTransport *ghinstallation.AppsTransport // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -112,6 +118,17 @@ func New(opts ...ClientOpt) (*client, error) { Scopes: githubScopes, } + if c.config.GithubAppID != 0 && len(c.config.GithubAppPrivateKey) > 0 { + c.Logger.Infof("sourcing private key from path: %s", c.config.GithubAppPrivateKey) + transport, err := ghinstallation.NewAppsTransportKeyFromFile(http.DefaultTransport, c.config.GithubAppID, c.config.GithubAppPrivateKey) + if err != nil { + return nil, err + } + + transport.BaseURL = c.config.API + c.AppsTransport = transport + } + return c, nil } @@ -167,3 +184,40 @@ func (c *client) newClientToken(token string) *github.Client { return github } + +// helper function to return the GitHub App token. +func (c *client) newGithubAppToken(r *library.Repo) *github.Client { + // create a github client based off the existing GitHub App configuration + client, err := github.NewClient(&http.Client{Transport: c.AppsTransport}).WithEnterpriseURLs(c.config.API, c.config.API) + if err != nil { + panic(err) + } + + // list all installations (a.k.a. orgs) where the GitHub App is installed + installations, _, err := client.Apps.ListInstallations(context.Background(), &github.ListOptions{}) + if err != nil { + panic(err) + } + + var id int64 + // iterate through the list of installations + for _, install := range installations { + // find the installation that matches the org for the repo + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + id = install.GetID() + } + } + + // failsafe in case the repo does not belong to an org where the GitHub App is installed + if id == 0 { + panic(err) + } + + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(context.Background(), id, &github.InstallationTokenOptions{}) + if err != nil { + panic(err) + } + + return c.newClientToken(t.GetToken()) +} From b011b7c1ad7481bd0358f3835546c060d8e359e1 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 27 Feb 2024 14:31:10 -0600 Subject: [PATCH 3/4] feat(scm): initial checks code --- scm/github/repo.go | 39 +++++++++++++++++++++++++++++++++++++++ scm/service.go | 4 ++++ 2 files changed, 43 insertions(+) diff --git a/scm/github/repo.go b/scm/github/repo.go index d53bc8a87..013886838 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -585,3 +585,42 @@ func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo return data.GetName(), data.GetCommit().GetSHA(), nil } + +// CreateChecks defines a function that does stuff... +func (c *client) CreateChecks(ctx context.Context, r *library.Repo, s *library.Step, branch string) (int64, error) { + // create client from GitHub App + client := c.newGithubAppToken(r) + + opts := github.CreateCheckRunOptions{ + // TODO: add step name? + Name: fmt.Sprintf("vela-%s-%s", branch, s.GetName()), + HeadSHA: branch, + } + + check, _, err := client.Checks.CreateCheckRun(ctx, r.GetOrg(), r.GetName(), opts) + if err != nil { + return 0, err + } + + return check.GetID(), nil +} + +// UpdateChecks defines a function that does stuff... +func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, id int64, branch string) error { + // create client from GitHub App + client := c.newGithubAppToken(r) + + opts := github.UpdateCheckRunOptions{ + // TODO: add step name? + Name: fmt.Sprintf("vela-%s-%s", branch, s.GetName()), + Status: github.String("completed"), + Conclusion: github.String("success"), + } + + _, _, err := client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), id, opts) + if err != nil { + return err + } + + return nil +} diff --git a/scm/service.go b/scm/service.go index 074495cee..c9d770e16 100644 --- a/scm/service.go +++ b/scm/service.go @@ -132,6 +132,10 @@ type Service interface { // a repository file's html_url. GetHTMLURL(context.Context, *library.User, string, string, string, string) (string, error) + // TODO: add comments + CreateChecks(context.Context, *library.Repo, *library.Step, string) (int64, error) + UpdateChecks(context.Context, *library.Repo, *library.Step, int64, string) error + // Webhook SCM Interface Functions // ProcessWebhook defines a function that From d4efd6b7f47cb21e6d008a1762ac2b2a40649304 Mon Sep 17 00:00:00 2001 From: JordanBrockopp Date: Tue, 27 Feb 2024 15:01:16 -0600 Subject: [PATCH 4/4] chore: push up updates --- api/build/plan.go | 6 ++++-- api/step/plan.go | 17 ++++++++++++--- api/step/update.go | 11 ++++++++++ scm/github/repo.go | 53 +++++++++++++++++++++++++++++++++++++--------- scm/service.go | 4 ++-- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/api/build/plan.go b/api/build/plan.go index 1c546e095..9d49eaffd 100644 --- a/api/build/plan.go +++ b/api/build/plan.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/api/service" "github.com/go-vela/server/api/step" "github.com/go-vela/server/database" @@ -19,7 +21,7 @@ import ( // and services, for the build in the configured backend. // TODO: // - return build and error. -func PlanBuild(ctx context.Context, database database.Interface, p *pipeline.Build, b *library.Build, r *library.Repo) error { +func PlanBuild(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *library.Build, r *library.Repo) error { // update fields in build object b.SetCreated(time.Now().UTC().Unix()) @@ -49,7 +51,7 @@ func PlanBuild(ctx context.Context, database database.Interface, p *pipeline.Bui } // plan all steps for the build - steps, err := step.PlanSteps(ctx, database, p, b) + steps, err := step.PlanSteps(ctx, database, scm, p, b, r) if err != nil { // clean up the objects from the pipeline in the database CleanBuild(ctx, database, b, services, steps, err) diff --git a/api/step/plan.go b/api/step/plan.go index 77f22adcb..6d77c8b5e 100644 --- a/api/step/plan.go +++ b/api/step/plan.go @@ -7,6 +7,8 @@ import ( "fmt" "time" + "github.com/go-vela/server/scm" + "github.com/go-vela/server/database" "github.com/go-vela/types/constants" "github.com/go-vela/types/library" @@ -16,7 +18,7 @@ import ( // PlanSteps is a helper function to plan all steps // in the build for execution. This creates the steps // for the build in the configured backend. -func PlanSteps(ctx context.Context, database database.Interface, p *pipeline.Build, b *library.Build) ([]*library.Step, error) { +func PlanSteps(ctx context.Context, database database.Interface, scm scm.Service, p *pipeline.Build, b *library.Build, r *library.Repo) ([]*library.Step, error) { // variable to store planned steps steps := []*library.Step{} @@ -25,7 +27,7 @@ func PlanSteps(ctx context.Context, database database.Interface, p *pipeline.Bui // iterate through all steps for each pipeline stage for _, step := range stage.Steps { // create the step object - s, err := planStep(ctx, database, b, step, stage.Name) + s, err := planStep(ctx, database, scm, b, r, step, stage.Name) if err != nil { return steps, err } @@ -47,7 +49,7 @@ func PlanSteps(ctx context.Context, database database.Interface, p *pipeline.Bui return steps, nil } -func planStep(ctx context.Context, database database.Interface, b *library.Build, c *pipeline.Container, stage string) (*library.Step, error) { +func planStep(ctx context.Context, database database.Interface, scm scm.Service, b *library.Build, r *library.Repo, c *pipeline.Container, stage string) (*library.Step, error) { // create the step object s := new(library.Step) s.SetBuildID(b.GetID()) @@ -59,6 +61,15 @@ func planStep(ctx context.Context, database database.Interface, b *library.Build s.SetStatus(constants.StatusPending) s.SetCreated(time.Now().UTC().Unix()) + id, err := scm.CreateChecks(ctx, r, b.GetCommit(), s.GetName()) + if err != nil { + // TODO: make this error more meaningful + return nil, err + } + + // TODO: have to store the check ID somewhere + s.SetCheckID(id) + // send API call to create the step s, err := database.CreateStep(ctx, s) if err != nil { diff --git a/api/step/update.go b/api/step/update.go index e049e0ad1..cdd3f2bd9 100644 --- a/api/step/update.go +++ b/api/step/update.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" + "github.com/go-vela/server/scm" + "github.com/gin-gonic/gin" "github.com/go-vela/server/database" "github.com/go-vela/server/router/middleware/build" @@ -155,5 +157,14 @@ func UpdateStep(c *gin.Context) { return } + err = scm.FromContext(c).UpdateChecks(ctx, r, s, b.GetCommit()) + if err != nil { + retErr := fmt.Errorf("unable to set step check %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + c.JSON(http.StatusOK, s) } diff --git a/scm/github/repo.go b/scm/github/repo.go index 013886838..48e837f6f 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -587,14 +587,13 @@ func (c *client) GetBranch(ctx context.Context, u *library.User, r *library.Repo } // CreateChecks defines a function that does stuff... -func (c *client) CreateChecks(ctx context.Context, r *library.Repo, s *library.Step, branch string) (int64, error) { +func (c *client) CreateChecks(ctx context.Context, r *library.Repo, commit, step string) (int64, error) { // create client from GitHub App client := c.newGithubAppToken(r) opts := github.CreateCheckRunOptions{ - // TODO: add step name? - Name: fmt.Sprintf("vela-%s-%s", branch, s.GetName()), - HeadSHA: branch, + Name: fmt.Sprintf("vela-%s-%s", commit, step), + HeadSHA: commit, } check, _, err := client.Checks.CreateCheckRun(ctx, r.GetOrg(), r.GetName(), opts) @@ -606,18 +605,52 @@ func (c *client) CreateChecks(ctx context.Context, r *library.Repo, s *library.S } // UpdateChecks defines a function that does stuff... -func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, id int64, branch string) error { +func (c *client) UpdateChecks(ctx context.Context, r *library.Repo, s *library.Step, commit string) error { // create client from GitHub App client := c.newGithubAppToken(r) + var ( + conclusion string + status string + ) + // set the conclusion and status for the step check depending on what the status of the step is + switch s.GetStatus() { + case constants.StatusPending: + conclusion = "neutral" + status = "queued" + case constants.StatusPendingApproval: + conclusion = "action_required" + status = "queued" + case constants.StatusRunning: + conclusion = "neutral" + status = "in_progress" + case constants.StatusSuccess: + conclusion = "success" + status = "completed" + case constants.StatusFailure: + conclusion = "failure" + status = "completed" + case constants.StatusCanceled: + conclusion = "cancelled" + status = "completed" + case constants.StatusKilled: + conclusion = "cancelled" + status = "completed" + case constants.StatusSkipped: + conclusion = "skipped" + status = "completed" + default: + conclusion = "neutral" + status = "completed" + } + opts := github.UpdateCheckRunOptions{ - // TODO: add step name? - Name: fmt.Sprintf("vela-%s-%s", branch, s.GetName()), - Status: github.String("completed"), - Conclusion: github.String("success"), + Name: fmt.Sprintf("vela-%s-%s", commit, s.GetName()), + Conclusion: github.String(conclusion), + Status: github.String(status), } - _, _, err := client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), id, opts) + _, _, err := client.Checks.UpdateCheckRun(ctx, r.GetOrg(), r.GetName(), s.GetCheckID(), opts) if err != nil { return err } diff --git a/scm/service.go b/scm/service.go index c9d770e16..daf3ff7ed 100644 --- a/scm/service.go +++ b/scm/service.go @@ -133,8 +133,8 @@ type Service interface { GetHTMLURL(context.Context, *library.User, string, string, string, string) (string, error) // TODO: add comments - CreateChecks(context.Context, *library.Repo, *library.Step, string) (int64, error) - UpdateChecks(context.Context, *library.Repo, *library.Step, int64, string) error + CreateChecks(context.Context, *library.Repo, string, string) (int64, error) + UpdateChecks(context.Context, *library.Repo, *library.Step, string) error // Webhook SCM Interface Functions