From f1495c35880bae03d7a5be4aa0c351579bfb2d81 Mon Sep 17 00:00:00 2001 From: Jad Chahed Date: Wed, 28 Aug 2024 19:24:46 +0200 Subject: [PATCH 01/19] feat: add execution Signed-off-by: Jad Chahed --- go/admin/admin.go | 2 ++ go/admin/templates/base.html | 14 ++++----- go/admin/templates/content.html | 51 +++++++++++++++++++++++++++++++-- go/admin/templates/sidebar.html | 5 ++-- go/server/api.go | 35 ++++++++++++++++++++++ go/server/cron.go | 12 +++++--- go/server/server.go | 5 ++-- 7 files changed, 105 insertions(+), 19 deletions(-) diff --git a/go/admin/admin.go b/go/admin/admin.go index f6642f6ae..868498896 100644 --- a/go/admin/admin.go +++ b/go/admin/admin.go @@ -42,6 +42,8 @@ const ( flagAdminAppSecret = "admin-gh-app-secret" ) +var workloads = []string{"OLTP", "OLTP-READONLY", "OLTP-SET", "TPCC", "TPCC_FK", "TPCC_FK_UNMANAGED", "TPCC_UNSHARDED"} + type Admin struct { port string router *gin.Engine diff --git a/go/admin/templates/base.html b/go/admin/templates/base.html index 29fc33c9d..949c05e32 100644 --- a/go/admin/templates/base.html +++ b/go/admin/templates/base.html @@ -13,26 +13,26 @@
-
- Loading Sidebar... +
+
{{ template "sidebar.html" . }}
- - + \ No newline at end of file diff --git a/go/admin/templates/content.html b/go/admin/templates/content.html index 521a2d5ae..5e41c10b5 100644 --- a/go/admin/templates/content.html +++ b/go/admin/templates/content.html @@ -1,4 +1,49 @@
-

Welcome to the Admin Dashboard

-

Use the sidebar to navigate through different sections of the dashboard.

-
+
+
+
+ + +
+
+ + +
+
+ +
+
+ + + + +
+
+ + + + +
+
+
+ + +
+ + +
+
\ No newline at end of file diff --git a/go/admin/templates/sidebar.html b/go/admin/templates/sidebar.html index f62fbfedf..97cc34a88 100644 --- a/go/admin/templates/sidebar.html +++ b/go/admin/templates/sidebar.html @@ -6,12 +6,11 @@

arewefastyet

diff --git a/go/server/api.go b/go/server/api.go index cf11de6b3..28643cbf5 100644 --- a/go/server/api.go +++ b/go/server/api.go @@ -27,6 +27,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/google/uuid" "github.com/vitessio/arewefastyet/go/exec" "github.com/vitessio/arewefastyet/go/tools/git" "github.com/vitessio/arewefastyet/go/tools/github" @@ -536,3 +537,37 @@ func (s *Server) getHistory(c *gin.Context) { c.JSON(http.StatusOK, results) } + +func (s *Server) addExecutions(c *gin.Context) { + source := c.PostForm("source") + sha := c.PostForm("sha") + workloads := c.PostFormArray("workloads") + numberOfExecutions := c.PostForm("numberOfExecutions") + + if source == "" || sha == "" || len(workloads) == 0 || numberOfExecutions == "" { + c.JSON(http.StatusBadRequest, &ErrorAPI{Error: "missing argument"}) + return + } + + if len(workloads) == 1 && workloads[0] == "all" { + workloads = s.workloads + } + execs, err := strconv.Atoi(numberOfExecutions) + if err != nil { + c.JSON(http.StatusBadRequest, &ErrorAPI{Error: "numberOfExecutions must be an integer"}) + return + } + newElements := make([]*executionQueueElement, 0, execs*len(workloads)) + + for _, workload := range workloads { + for i := 0; i < execs; i++ { + elem := s.createSimpleExecutionQueueElement(s.benchmarkConfig[strings.ToLower(workload)], source, sha, workload, string(macrobench.Gen4Planner), false, 0, git.Version{}) + elem.identifier.UUID = uuid.NewString() + newElements = append(newElements, elem) + } + } + + s.appendToQueue(newElements) + + c.JSON(http.StatusOK, "ok") +} diff --git a/go/server/cron.go b/go/server/cron.go index 7cf2cbc1b..48ce9be83 100644 --- a/go/server/cron.go +++ b/go/server/cron.go @@ -40,10 +40,10 @@ type ( executionIdentifier struct { GitRef, Source, Workload, PlannerVersion string - PullNb int - PullBaseRef string - Version git.Version - UUID string + PullNb int + PullBaseRef string + Version git.Version + UUID string } executionQueue map[executionIdentifier]*executionQueueElement @@ -172,6 +172,10 @@ func (s *Server) addToQueue(element *executionQueueElement) { } // Add all the elements to the queue + s.appendToQueue(execElements) +} + +func (s *Server) appendToQueue(execElements []*executionQueueElement) { for _, execElement := range execElements { // Check if the exact same benchmark is already in the queue _, found := queue[execElement.identifier] diff --git a/go/server/server.go b/go/server/server.go index 50fc42f3e..bd09993ef 100644 --- a/go/server/server.go +++ b/go/server/server.go @@ -223,8 +223,8 @@ func (s *Server) Run() error { s.router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, - AllowMethods: []string{"GET"}, - AllowHeaders: []string{"Origin"}, + AllowMethods: []string{"GET", "POST"}, + AllowHeaders: []string{"Origin", "hx-request", "hx-current-url"}, ExposeHeaders: []string{"Content-Length"}, AllowCredentials: true, MaxAge: 12 * time.Hour, @@ -249,6 +249,7 @@ func (s *Server) Run() error { s.router.GET("/api/status/stats", s.getStatusStats) s.router.GET("/api/run/request", s.requestRun) s.router.GET("/api/run/delete", s.deleteRun) + s.router.POST("/api/executions/add", s.addExecutions) return s.router.Run(":" + s.port) } From 1f765bdfbe5e01749f9ed72d55dc7de4795b3184 Mon Sep 17 00:00:00 2001 From: Jad Chahed Date: Wed, 28 Aug 2024 20:10:16 +0200 Subject: [PATCH 02/19] feat: gh token in cookies Signed-off-by: Jad Chahed --- go/admin/admin.go | 5 ----- go/admin/api.go | 20 ++++++++++++-------- go/admin/templates/content.html | 2 +- go/server/server.go | 2 +- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go/admin/admin.go b/go/admin/admin.go index 868498896..85d5d57c7 100644 --- a/go/admin/admin.go +++ b/go/admin/admin.go @@ -26,8 +26,6 @@ import ( "time" "github.com/gin-contrib/cors" - "github.com/gin-contrib/sessions" - "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -115,9 +113,6 @@ func (a *Admin) Run() error { a.Mode.SetGin() a.router = gin.Default() - store := cookie.NewStore([]byte("secret")) - a.router.Use(sessions.Sessions("mysession", store)) - a.router.Static("/assets", filepath.Join(basepath, "assets")) a.router.LoadHTMLGlob(filepath.Join(basepath, "templates/*")) diff --git a/go/admin/api.go b/go/admin/api.go index 3b2ed66a7..db1fd3b9d 100644 --- a/go/admin/api.go +++ b/go/admin/api.go @@ -22,8 +22,8 @@ import ( "context" "net/http" "strings" + "time" - "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" goGithub "github.com/google/go-github/github" "github.com/labstack/gommon/random" @@ -51,14 +51,20 @@ func (a *Admin) login(c *gin.Context) { } func (a *Admin) dashboard(c *gin.Context) { - a.render(c, gin.H{}, "base.html") + user, err := c.Cookie("ghtoken") + if err != nil { + c.Abort() + return + } + a.render(c, gin.H{ + "ghtoken": user, + }, "base.html") } func (a *Admin) authMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - session := sessions.Default(c) - user := session.Get("user") - if user == nil { + _, err := c.Cookie("ghtoken") + if err != nil { // User not authenticated, redirect to login c.Redirect(http.StatusSeeOther, "/admin/login") c.Abort() @@ -115,9 +121,7 @@ func (a *Admin) handleGitHubCallback(c *gin.Context) { } if isMaintainer { - session := sessions.Default(c) - session.Set("user", user.GetLogin()) - _ = session.Save() + c.SetCookie("ghtoken", token.AccessToken, int(token.Expiry.Sub(time.Now()).Seconds()), "/", "localhost", true, true) c.Redirect(http.StatusSeeOther, "/admin/dashboard") } else { diff --git a/go/admin/templates/content.html b/go/admin/templates/content.html index 5e41c10b5..1017c3025 100644 --- a/go/admin/templates/content.html +++ b/go/admin/templates/content.html @@ -42,7 +42,7 @@ type="number" name="numberOfExecutions" value="1" min="1" max="10">
- diff --git a/go/server/server.go b/go/server/server.go index bd09993ef..e6d25f0d3 100644 --- a/go/server/server.go +++ b/go/server/server.go @@ -224,7 +224,7 @@ func (s *Server) Run() error { s.router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, AllowMethods: []string{"GET", "POST"}, - AllowHeaders: []string{"Origin", "hx-request", "hx-current-url"}, + AllowHeaders: []string{"Origin", "hx-request", "hx-current-url", "admin-user"}, ExposeHeaders: []string{"Content-Length"}, AllowCredentials: true, MaxAge: 12 * time.Hour, From caac9f8ab68896e580775a29fe0100680edcf6f2 Mon Sep 17 00:00:00 2001 From: Jad Chahed Date: Fri, 30 Aug 2024 15:56:45 +0200 Subject: [PATCH 03/19] feat: encrypted gh token Signed-off-by: Jad Chahed --- go/admin/admin.go | 14 ++++- go/admin/api.go | 87 +++++++++++++++++++++++++++--- go/admin/templates/content.html | 2 +- go/admin/templates/executions.html | 36 ------------- go/admin/templates/header.html | 3 +- go/admin/templates/sidebar.html | 6 ++- go/server/api.go | 41 ++++++++++---- go/server/server.go | 8 ++- go/tools/server/utils.go | 73 +++++++++++++++++++++++++ 9 files changed, 207 insertions(+), 63 deletions(-) delete mode 100644 go/admin/templates/executions.html diff --git a/go/admin/admin.go b/go/admin/admin.go index 85d5d57c7..aec1632b2 100644 --- a/go/admin/admin.go +++ b/go/admin/admin.go @@ -26,6 +26,8 @@ import ( "time" "github.com/gin-contrib/cors" + "github.com/gin-contrib/sessions" + "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -38,6 +40,7 @@ const ( flagMode = "admin-mode" flagAdminAppId = "admin-gh-app-id" flagAdminAppSecret = "admin-gh-app-secret" + flagGhAuth = "auth" ) var workloads = []string{"OLTP", "OLTP-READONLY", "OLTP-SET", "TPCC", "TPCC_FK", "TPCC_FK_UNMANAGED", "TPCC_UNSHARDED"} @@ -48,6 +51,7 @@ type Admin struct { ghAppId string ghAppSecret string + ghTokenSalt string dbCfg *psdb.Config dbClient *psdb.Client @@ -60,11 +64,13 @@ func (a *Admin) AddToCommand(cmd *cobra.Command) { cmd.Flags().Var(&a.Mode, flagMode, "Specify the mode on which the server will run") cmd.Flags().StringVar(&a.ghAppId, flagAdminAppId, "", "The ID of the GitHub App") cmd.Flags().StringVar(&a.ghAppSecret, flagAdminAppSecret, "", "The secret of the GitHub App") + cmd.Flags().StringVar(&a.ghTokenSalt, flagGhAuth, "", "The salt string to salt the GitHub Token") _ = viper.BindPFlag(flagPort, cmd.Flags().Lookup(flagPort)) _ = viper.BindPFlag(flagMode, cmd.Flags().Lookup(flagMode)) _ = viper.BindPFlag(flagAdminAppId, cmd.Flags().Lookup(flagAdminAppId)) _ = viper.BindPFlag(flagAdminAppSecret, cmd.Flags().Lookup(flagAdminAppSecret)) + _ = viper.BindPFlag(flagGhAuth, cmd.Flags().Lookup(flagGhAuth)) if a.dbCfg == nil { a.dbCfg = &psdb.Config{} @@ -113,15 +119,18 @@ func (a *Admin) Run() error { a.Mode.SetGin() a.router = gin.Default() + store := cookie.NewStore([]byte("secret")) + a.router.Use(sessions.Sessions("admin-session", store)) + a.router.Static("/assets", filepath.Join(basepath, "assets")) a.router.LoadHTMLGlob(filepath.Join(basepath, "templates/*")) a.router.Use(cors.New(cors.Config{ AllowOrigins: []string{"*"}, - AllowMethods: []string{"GET"}, + AllowMethods: []string{"GET", "POST"}, AllowHeaders: []string{"Origin"}, - ExposeHeaders: []string{"Content-Length"}, + ExposeHeaders: []string{"Content-Length", "Content-Type"}, AllowCredentials: true, MaxAge: 12 * time.Hour, })) @@ -130,6 +139,7 @@ func (a *Admin) Run() error { a.router.GET("/admin", a.login) a.router.GET("/admin/login", a.handleGitHubLogin) a.router.GET("/admin/auth/callback", a.handleGitHubCallback) + a.router.POST("/admin/executions/add", a.handleExecutionsAdd) a.router.GET("/admin/dashboard", a.authMiddleware(), a.dashboard) return a.router.Run(":" + a.port) diff --git a/go/admin/api.go b/go/admin/api.go index db1fd3b9d..c31f52b63 100644 --- a/go/admin/api.go +++ b/go/admin/api.go @@ -19,7 +19,9 @@ package admin import ( + "bytes" "context" + "encoding/json" "net/http" "strings" "time" @@ -39,6 +41,7 @@ var ( RedirectURL: "http://localhost:8081/admin/auth/callback", } oauthStateString = random.String(10) // A random string to protect against CSRF attacks + client *goGithub.Client ) const ( @@ -46,19 +49,20 @@ const ( arewefastyetTeamGitHub = "arewefastyet" ) +type ExecutionRequest struct { + Auth string `json:"auth"` + Source string `json:"source"` + SHA string `json:"sha"` + Workloads []string `json:"workloads"` + NumberOfExecutions string `json:"number_of_executions"` +} + func (a *Admin) login(c *gin.Context) { a.render(c, gin.H{}, "login.html") } func (a *Admin) dashboard(c *gin.Context) { - user, err := c.Cookie("ghtoken") - if err != nil { - c.Abort() - return - } - a.render(c, gin.H{ - "ghtoken": user, - }, "base.html") + a.render(c, gin.H{}, "base.html") } func (a *Admin) authMiddleware() gin.HandlerFunc { @@ -153,3 +157,70 @@ func (a *Admin) checkUserOrgMembership(client *goGithub.Client, username, orgNam } return isMember, nil } + +func (a *Admin) handleExecutionsAdd(c *gin.Context) { + slog.Info("Adding execution") + token, err := c.Cookie("ghtoken") + source := c.PostForm("source") + sha := c.PostForm("sha") + workloads := c.PostFormArray("workloads") + numberOfExecutions := c.PostForm("numberOfExecutions") + + if err != nil { + slog.Error("Failed to get token from cookie: ", err) + c.AbortWithStatus(http.StatusUnauthorized) + return + } + + encryptedToken := server.Encrypt(token, a.ghTokenSalt) + + requestPayload := ExecutionRequest{ + Auth: encryptedToken, + Source: source, + SHA: sha, + Workloads: workloads, + NumberOfExecutions: numberOfExecutions, + } + + jsonData, err := json.Marshal(requestPayload) + + slog.Infof("Request payload: %s", jsonData) + + if err != nil { + slog.Error("Failed to marshal request payload: ", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to marshal request payload"}) + return + } + + serverAPIURL := "http://localhost:8080/api/executions/add" + // make a body with "admin-user": {encryptedToken}, "source": {source}, "sha": {sha}, "workloads": workloads, "numberOfExecutions": {numberOfExecutions}} + + req, err := http.NewRequest("POST", serverAPIURL, bytes.NewBuffer(jsonData)) + + if err != nil { + slog.Error("Failed to create new HTTP request: ", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create request to server API"}) + return + } + + req.Header.Set("Content-Type", "application/json") + + // Execute the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + slog.Error("Failed to send request to server API: ", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to send request to server API"}) + return + } + defer resp.Body.Close() + + // Handle the response from the server API + if resp.StatusCode != http.StatusOK { + slog.Error("Server API returned an error: ", resp.Status) + c.JSON(resp.StatusCode, gin.H{"error": "Failed to process request on server API"}) + return + } + + c.JSON(http.StatusOK, gin.H{"message": "Request forwarded to server API"}) +} diff --git a/go/admin/templates/content.html b/go/admin/templates/content.html index 1017c3025..ca8a55843 100644 --- a/go/admin/templates/content.html +++ b/go/admin/templates/content.html @@ -42,7 +42,7 @@ type="number" name="numberOfExecutions" value="1" min="1" max="10"> - diff --git a/go/admin/templates/executions.html b/go/admin/templates/executions.html deleted file mode 100644 index 5a985748e..000000000 --- a/go/admin/templates/executions.html +++ /dev/null @@ -1,36 +0,0 @@ -
-

Add new executions

-
-
- - -
- -
-
- -
-
- -
- -
-
- -
- -
- - -
- -
- -
-
-
diff --git a/go/admin/templates/header.html b/go/admin/templates/header.html index 177d4edd3..7d654a022 100644 --- a/go/admin/templates/header.html +++ b/go/admin/templates/header.html @@ -1,3 +1,4 @@ -
+

Admin Dashboard

+ arewefastyet website
diff --git a/go/admin/templates/sidebar.html b/go/admin/templates/sidebar.html index 97cc34a88..2fef25d76 100644 --- a/go/admin/templates/sidebar.html +++ b/go/admin/templates/sidebar.html @@ -1,7 +1,9 @@