Skip to content

Commit

Permalink
Automatically track perf of all SQL queries (and rework perf)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvisness
bvisness committed Jan 20, 2025
1 parent aafd294 commit 27ac3b1
Show file tree
Hide file tree
Showing 34 changed files with 536 additions and 516 deletions.
5 changes: 4 additions & 1 deletion src/auth/session.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,10 @@ func CreateSession(ctx context.Context, conn *pgxpool.Pool, username string) (*m
}

_, err := conn.Exec(ctx,
"INSERT INTO session (id, username, expires_at, csrf_token) VALUES ($1, $2, $3, $4)",
`
---- Create session
INSERT INTO session (id, username, expires_at, csrf_token) VALUES ($1, $2, $3, $4)
`,
session.ID, session.Username, session.ExpiresAt, session.CSRFToken,
)
if err != nil {
Expand Down
95 changes: 6 additions & 89 deletions src/db/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,107 +12,24 @@ import (
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/utils"
"github.com/google/uuid"
zerologadapter "github.com/jackc/pgx-zerolog"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/jackc/pgx/v5/tracelog"
"github.com/rs/zerolog/log"
)

/*
A general error to be used when no results are found. This is the error returned
by QueryOne, and can generally be used by other database helpers that fetch a single
result but find nothing.
*/
var NotFound = errors.New("not found")

// This interface should match both a direct pgx connection or a pgx transaction.
type ConnOrTx interface {
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)
CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)

// Both raw database connections and transactions in pgx can begin/commit
// transactions. For database connections it does the obvious thing; for
// transactions it creates a "pseudo-nested transaction" but conceptually
// works the same. See the documentation of pgx.Tx.Begin.
Begin(ctx context.Context) (pgx.Tx, error)
}

var pgTypeMap = pgtype.NewMap()

func init() {
// NOTE(asaf): Need to initialize it here to avoid potential race conditions later
pgTypeMap.TypeForValue(nil)
}

// Creates a new connection to the HMN database.
// This connection is not safe for concurrent use.
func NewConn() *pgx.Conn {
return NewConnWithConfig(config.PostgresConfig{})
}

func NewConnWithConfig(cfg config.PostgresConfig) *pgx.Conn {
cfg = overrideDefaultConfig(cfg)

pgcfg, err := pgx.ParseConfig(cfg.DSN())

pgcfg.Tracer = &tracelog.TraceLog{
Logger: zerologadapter.NewLogger(log.Logger),
LogLevel: cfg.LogLevel,
}

conn, err := pgx.ConnectConfig(context.Background(), pgcfg)
if err != nil {
panic(oops.New(err, "failed to connect to database"))
}

return conn
}

// Creates a connection pool for the HMN database.
// The resulting pool is safe for concurrent use.
func NewConnPool() *pgxpool.Pool {
return NewConnPoolWithConfig(config.PostgresConfig{})
}

func NewConnPoolWithConfig(cfg config.PostgresConfig) *pgxpool.Pool {
cfg = overrideDefaultConfig(cfg)

pgcfg, err := pgxpool.ParseConfig(cfg.DSN())

pgcfg.MinConns = cfg.MinConn
pgcfg.MaxConns = cfg.MaxConn
pgcfg.ConnConfig.Tracer = &tracelog.TraceLog{
Logger: zerologadapter.NewLogger(log.Logger),
LogLevel: cfg.LogLevel,
}

conn, err := pgxpool.NewWithConfig(context.Background(), pgcfg)
if err != nil {
panic(oops.New(err, "failed to create database connection pool"))
}

return conn
}

func overrideDefaultConfig(cfg config.PostgresConfig) config.PostgresConfig {
return config.PostgresConfig{
User: utils.OrDefault(cfg.User, config.Config.Postgres.User),
Password: utils.OrDefault(cfg.Password, config.Config.Postgres.Password),
Hostname: utils.OrDefault(cfg.Hostname, config.Config.Postgres.Hostname),
Port: utils.OrDefault(cfg.Port, config.Config.Postgres.Port),
DbName: utils.OrDefault(cfg.DbName, config.Config.Postgres.DbName),
LogLevel: utils.OrDefault(cfg.LogLevel, config.Config.Postgres.LogLevel),
MinConn: utils.OrDefault(cfg.MinConn, config.Config.Postgres.MinConn),
MaxConn: utils.OrDefault(cfg.MaxConn, config.Config.Postgres.MaxConn),
}
}
/*
A general error to be used when no results are found. This is the error returned
by QueryOne, and can generally be used by other database helpers that fetch a single
result but find nothing.
*/
var NotFound = errors.New("not found")

/*
Performs a SQL query and returns a slice of all the result rows. The query is just plain SQL, but make sure to read the package documentation for details. You must explicitly provide the type argument - this is how it knows what Go type to map the results to, and it cannot be inferred.
Expand Down
4 changes: 4 additions & 0 deletions src/db/query_builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ func (qb *QueryBuilder) Add(sql string, args ...any) {
qb.sql.WriteString("\n")
}

func (qb *QueryBuilder) AddName(name string) {
qb.Add("---- " + name + "\n")
}

func (qb *QueryBuilder) String() string {
return qb.sql.String()
}
Expand Down
45 changes: 22 additions & 23 deletions src/email/email.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ func SendRegistrationEmail(
destination string,
perf *perf.RequestPerf,
) error {
perf.StartBlock("EMAIL", "Registration email")
defer perf.StartBlock("EMAIL", "Registration email").End()

perf.StartBlock("EMAIL", "Rendering template")
b1 := perf.StartBlock("EMAIL", "Rendering template")
defer b1.End()
contents, err := renderTemplate("email_registration.html", RegistrationEmailData{
Name: toName,
HomepageUrl: hmnurl.BuildHomepage(),
Expand All @@ -45,16 +46,15 @@ func SendRegistrationEmail(
if err != nil {
return err
}
perf.EndBlock()
b1.End()

perf.StartBlock("EMAIL", "Sending email")
b2 := perf.StartBlock("EMAIL", "Sending email")
defer b2.End()
err = sendMail(toAddress, toName, "[Handmade Network] Registration confirmation", contents)
if err != nil {
return oops.New(err, "Failed to send email")
}
perf.EndBlock()

perf.EndBlock()
b2.End()

return nil
}
Expand All @@ -73,9 +73,10 @@ func SendExistingAccountEmail(
destination string,
perf *perf.RequestPerf,
) error {
perf.StartBlock("EMAIL", "Existing account email")
defer perf.StartBlock("EMAIL", "Existing account email").End()

perf.StartBlock("EMAIL", "Rendering template")
b1 := perf.StartBlock("EMAIL", "Rendering template")
defer b1.End()
contents, err := renderTemplate("email_account_existing.html", ExistingAccountEmailData{
Name: toName,
Username: username,
Expand All @@ -85,16 +86,15 @@ func SendExistingAccountEmail(
if err != nil {
return err
}
perf.EndBlock()
b1.End()

perf.StartBlock("EMAIL", "Sending email")
b2 := perf.StartBlock("EMAIL", "Sending email")
defer b2.End()
err = sendMail(toAddress, toName, "[Handmade Network] You already have an account!", contents)
if err != nil {
return oops.New(err, "Failed to send email")
}
perf.EndBlock()

perf.EndBlock()
b2.End()

return nil
}
Expand All @@ -106,9 +106,10 @@ type PasswordResetEmailData struct {
}

func SendPasswordReset(toAddress string, toName string, username string, resetToken string, expiration time.Time, perf *perf.RequestPerf) error {
perf.StartBlock("EMAIL", "Password reset email")
defer perf.StartBlock("EMAIL", "Password reset email").End()

perf.StartBlock("EMAIL", "Rendering template")
b1 := perf.StartBlock("EMAIL", "Rendering template")
defer b1.End()
contents, err := renderTemplate("email_password_reset.html", PasswordResetEmailData{
Name: toName,
DoPasswordResetUrl: hmnurl.BuildDoPasswordReset(username, resetToken),
Expand All @@ -117,16 +118,15 @@ func SendPasswordReset(toAddress string, toName string, username string, resetTo
if err != nil {
return err
}
perf.EndBlock()
b1.End()

perf.StartBlock("EMAIL", "Sending email")
b2 := perf.StartBlock("EMAIL", "Sending email")
defer b2.End()
err = sendMail(toAddress, toName, "[Handmade Network] Your password reset request", contents)
if err != nil {
return oops.New(err, "Failed to send email")
}
perf.EndBlock()

perf.EndBlock()
b2.End()

return nil
}
Expand All @@ -142,8 +142,7 @@ type TimeMachineEmailData struct {
}

func SendTimeMachineEmail(profileUrl, username, userEmail, discordUsername string, mediaUrls []string, deviceInfo, description string, perf *perf.RequestPerf) error {
perf.StartBlock("EMAIL", "Time machine email")
defer perf.EndBlock()
defer perf.StartBlock("EMAIL", "Time machine email").End()

contents, err := renderTemplate("email_time_machine.html", TimeMachineEmailData{
ProfileUrl: profileUrl,
Expand Down
1 change: 1 addition & 0 deletions src/hmndata/jams.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ func JamBySlug(slug string) Jam {
func FetchJamsForProject(ctx context.Context, dbConn db.ConnOrTx, user *models.User, projectId int) ([]*models.JamProject, error) {
jamProjects, err := db.Query[models.JamProject](ctx, dbConn,
`
---- Fetch jams for project
SELECT $columns
FROM jam_project
WHERE project_id = $1
Expand Down
30 changes: 18 additions & 12 deletions src/hmndata/project_helper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,7 @@ func FetchProjects(
currentUser *models.User,
q ProjectsQuery,
) ([]ProjectAndStuff, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("SQL", "Fetch projects")
defer perf.EndBlock()
defer perf.StartBlock(ctx, "PROJECT", "Fetch projects").End()

tx, err := dbConn.Begin(ctx)
if err != nil {
Expand All @@ -80,6 +78,7 @@ func FetchProjects(

// Fetch all valid projects (not yet subject to user permission checks)
var qb db.QueryBuilder
qb.AddName("Fetch projects")
if len(q.OrderBy) > 0 {
qb.Add(`SELECT * FROM (`)
}
Expand Down Expand Up @@ -174,6 +173,7 @@ func FetchProjects(
return nil, err
}

b := perf.StartBlock(ctx, "PROJECT", "Compute permissions")
var res []ProjectAndStuff
for i, p := range projectRows {
owners := projectOwners[i].Owners
Expand Down Expand Up @@ -232,6 +232,7 @@ func FetchProjects(
})
}
}
b.End()

err = tx.Commit(ctx)
if err != nil {
Expand All @@ -253,6 +254,8 @@ func FetchProject(
projectID int,
q ProjectsQuery,
) (ProjectAndStuff, error) {
defer perf.StartBlock(ctx, "PROJECT", "Fetch project").End()

q.ProjectIDs = []int{projectID}
q.Limit = 1
q.Offset = 0
Expand Down Expand Up @@ -281,6 +284,8 @@ func FetchProjectBySlug(
projectSlug string,
q ProjectsQuery,
) (ProjectAndStuff, error) {
defer perf.StartBlock(ctx, "PROJECT", "Fetch project by slug").End()

q.Slugs = []string{projectSlug}
q.Limit = 1
q.Offset = 0
Expand All @@ -303,9 +308,7 @@ func CountProjects(
currentUser *models.User,
q ProjectsQuery,
) (int, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("SQL", "Count projects")
defer perf.EndBlock()
defer perf.StartBlock(ctx, "PROJECT", "Count projects").End()

q.Limit = 0
q.Offset = 0
Expand Down Expand Up @@ -338,9 +341,7 @@ func FetchMultipleProjectsOwners(
dbConn db.ConnOrTx,
projectIds []int,
) ([]ProjectOwners, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("SQL", "Fetch owners for multiple projects")
defer perf.EndBlock()
defer perf.StartBlock(ctx, "PROJECT", "Fetch owners for multiple projects").End()

tx, err := dbConn.Begin(ctx)
if err != nil {
Expand All @@ -355,6 +356,7 @@ func FetchMultipleProjectsOwners(
}
userProjects, err := db.Query[userProject](ctx, tx,
`
---- Fetch user/project pairs
SELECT $columns
FROM user_project
WHERE project_id = ANY($1)
Expand Down Expand Up @@ -430,9 +432,7 @@ func FetchProjectOwners(
dbConn db.ConnOrTx,
projectId int,
) ([]*models.User, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("SQL", "Fetch owners for project")
defer perf.EndBlock()
defer perf.StartBlock(ctx, "PROJECT", "Fetch project owners").End()

projectOwners, err := FetchMultipleProjectsOwners(ctx, dbConn, []int{projectId})
if err != nil {
Expand All @@ -458,6 +458,8 @@ func SetProjectTag(
projectID int,
tagText string,
) (*models.Tag, error) {
defer perf.StartBlock(ctx, "PROJECT", "Set project tag").End()

tx, err := dbConn.Begin(ctx)
if err != nil {
return nil, oops.New(err, "failed to start transaction")
Expand All @@ -480,6 +482,7 @@ func SetProjectTag(
// Create a tag
tag, err := db.QueryOne[models.Tag](ctx, tx,
`
---- Create a new tag
INSERT INTO tag (text) VALUES ($1)
RETURNING $columns
`,
Expand All @@ -493,6 +496,7 @@ func SetProjectTag(
// Attach it to the project
_, err = tx.Exec(ctx,
`
---- Associate tag with project
UPDATE project
SET tag = $1
WHERE id = $2
Expand All @@ -506,6 +510,7 @@ func SetProjectTag(
// Update the text of an existing one
tag, err := db.QueryOne[models.Tag](ctx, tx,
`
---- Update the text of the existing tag
UPDATE tag
SET text = $1
WHERE id = (SELECT tag FROM project WHERE id = $2)
Expand All @@ -530,6 +535,7 @@ func SetProjectTag(
func UpdateSnippetLastPostedForAllProjects(ctx context.Context, dbConn db.ConnOrTx) error {
_, err := dbConn.Exec(ctx,
`
---- Update snippet_last_posted for everything
UPDATE project p SET (snippet_last_posted, all_last_updated) = (
SELECT
COALESCE(MAX(s."when"), 'epoch'),
Expand Down
Loading

0 comments on commit 27ac3b1

Please sign in to comment.