From f812b02ab0c89315af0e7ae6436327646a080bd5 Mon Sep 17 00:00:00 2001 From: Anup Cowkur Date: Wed, 28 Aug 2024 16:47:21 +0530 Subject: [PATCH] chore(backend): remove eventIds array from crash + anr groups closes #1027 Crash and ANR groups have a list of event ids associated with them which are stored as an array in the 'event_ids' column in postgres. Postgres fields have a limit of 1GB and this list can get quite large in high scale environments. To improve this situation, this commit: 1. Removes the event_ids column from postgres 2. Fetches event ids of crash + anr groups from clickhouse based on the event's fingeprint 3. Updates the ingest algorithm to no longer append event_ids to crash & anr groups and only update updated_at and first_seen_event timestamps 4. Updates the cleanup service to no longer delete event_ids from postgres on stale event cleanup --- backend/api/group/group.go | 154 +++++++++--- backend/api/measure/app.go | 220 ++++++++++++++++-- backend/api/measure/event.go | 13 +- backend/cleanup/cleanup/cleanup.go | 104 --------- ...reate_unhandled_exception_groups_table.sql | 2 - ...20231228044339_create_anr_groups_table.sql | 2 - 6 files changed, 329 insertions(+), 166 deletions(-) diff --git a/backend/api/group/group.go b/backend/api/group/group.go index 3f520d1eb..0c9e36f30 100644 --- a/backend/api/group/group.go +++ b/backend/api/group/group.go @@ -28,8 +28,8 @@ type ExceptionGroup struct { FileName string `json:"file_name" db:"file_name"` LineNumber int `json:"line_number" db:"line_number"` Fingerprint string `json:"fingerprint" db:"fingerprint"` - Count int `json:"count" db:"count"` - EventIDs []uuid.UUID `json:"event_ids,omitempty" db:"event_ids"` + Count int `json:"count"` + EventIDs []uuid.UUID `json:"event_ids,omitempty"` EventExceptions []event.EventException `json:"exception_events,omitempty"` Percentage float32 `json:"percentage_contribution"` FirstEventTime time.Time `json:"-" db:"first_event_timestamp"` @@ -46,8 +46,8 @@ type ANRGroup struct { FileName string `json:"file_name" db:"file_name"` LineNumber int `json:"line_number" db:"line_number"` Fingerprint string `json:"fingerprint" db:"fingerprint"` - Count int `json:"count" db:"count"` - EventIDs []uuid.UUID `json:"event_ids,omitempty" db:"event_ids"` + Count int `json:"count"` + EventIDs []uuid.UUID `json:"event_ids,omitempty"` EventANRs []event.EventANR `json:"anr_events,omitempty"` Percentage float32 `json:"percentage_contribution"` FirstEventTime time.Time `json:"-" db:"first_event_timestamp"` @@ -73,14 +73,13 @@ func (e ExceptionGroup) EventExists(id uuid.UUID) bool { }) } -// AppendEvent appends a new event id to the ExceptionGroup's -// events array. Additionally, if the event's timestamp is +// UpdateTimeStamps updates the updated_at timestamp of the +// ExceptionGroup. Additionally, if the event's timestamp is // older than the group's timestamp, then update the group's // timestamp. -func (e ExceptionGroup) AppendEvent(ctx context.Context, event *event.EventField, tx *pgx.Tx) (err error) { +func (e ExceptionGroup) UpdateTimeStamps(ctx context.Context, event *event.EventField, tx *pgx.Tx) (err error) { stmt := sqlf.PostgreSQL. Update("public.unhandled_exception_groups"). - SetExpr("event_ids", "array_append(event_ids, ?)", event.ID). Set("updated_at", time.Now()). Where("id = ?", e.ID) @@ -117,7 +116,6 @@ func (e *ExceptionGroup) Insert(ctx context.Context, tx *pgx.Tx) (err error) { Set("file_name", e.FileName). Set("line_number", e.LineNumber). Set("fingerprint", e.Fingerprint). - Set("event_ids", e.EventIDs). Set("first_event_timestamp", e.FirstEventTime) defer stmt.Close() @@ -150,14 +148,13 @@ func (a ANRGroup) EventExists(id uuid.UUID) bool { }) } -// AppendEvent appends a new event id to the ANRGroup's -// events array. Additionally, if the event's timestamp is +// UpdateTimeStamps updates the updated_at timestamp of the +// ANRGroup. Additionally, if the event's timestamp is // older than the group's timestamp, then update the group's // timestamp. -func (e ANRGroup) AppendEvent(ctx context.Context, event *event.EventField, tx *pgx.Tx) (err error) { +func (e ANRGroup) UpdateTimeStamps(ctx context.Context, event *event.EventField, tx *pgx.Tx) (err error) { stmt := sqlf.PostgreSQL. Update("public.anr_groups"). - SetExpr("event_ids", "array_append(event_ids, ?)", event.ID). Set("updated_at", time.Now()). Where("id = ?", e.ID) @@ -194,7 +191,6 @@ func (a *ANRGroup) Insert(ctx context.Context, tx *pgx.Tx) (err error) { Set("file_name", a.FileName). Set("line_number", a.LineNumber). Set("fingerprint", a.Fingerprint). - Set("event_ids", a.EventIDs). Set("first_event_timestamp", a.FirstEventTime) defer stmt.Close() @@ -262,6 +258,40 @@ func SortANRGroups(groups []ANRGroup) { // GetExceptionGroupsFromExceptionIds gets exception groups // matched by exception ids. func GetExceptionGroupsFromExceptionIds(ctx context.Context, eventIds []uuid.UUID) (exceptionGroups []ExceptionGroup, err error) { + // Get list of fingerprints and event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id, exception.fingerprint`). + Where(`id in (?)`, eventIds). + Where(`exception.fingerprint != ''`) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + fingerprints := make([]string, 0) + fingerprintSet := make(map[string]struct{}) + eventIdToFingerprint := make(map[uuid.UUID]string) + + for eventDataRows.Next() { + var eventID uuid.UUID + var fingerprint string + if err := eventDataRows.Scan(&eventID, &fingerprint); err != nil { + return nil, err + } + eventIdToFingerprint[eventID] = fingerprint + if _, exists := fingerprintSet[fingerprint]; !exists { + fingerprints = append(fingerprints, fingerprint) + fingerprintSet[fingerprint] = struct{}{} + } + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + // Query groups that match the obtained fingerprints stmt := sqlf.PostgreSQL. From(`public.unhandled_exception_groups`). Select(`id`). @@ -270,23 +300,73 @@ func GetExceptionGroupsFromExceptionIds(ctx context.Context, eventIds []uuid.UUI Select(`method_name`). Select(`file_name`). Select(`line_number`). - Select(`event_ids`). - // `&&` matches rows by list of uuids - Where(`event_ids && ?`, eventIds) + Select(`fingerprint`). + Where(`fingerprint = ANY(?)`, fingerprints) defer stmt.Close() rows, _ := server.Server.PgPool.Query(ctx, stmt.String(), stmt.Args()...) exceptionGroups, err = pgx.CollectRows(rows, pgx.RowToStructByNameLax[ExceptionGroup]) - err = rows.Err() + if err != nil { + return nil, err + } + if rows.Err() != nil { + return nil, rows.Err() + } - return + // Add event ids to obtained exception groups + fingerprintToGroup := make(map[string]*ExceptionGroup) + for i := range exceptionGroups { + fingerprintToGroup[exceptionGroups[i].Fingerprint] = &exceptionGroups[i] + } + + for eventID, fingerprint := range eventIdToFingerprint { + if group, ok := fingerprintToGroup[fingerprint]; ok { + group.EventIDs = append(group.EventIDs, eventID) + } + } + + return exceptionGroups, nil } -// GetANRGroupsFromANRIds gets anr groups -// matched by anr ids. +// GetANRGroupsFromANRIds gets ANR groups +// matched by ANR ids. func GetANRGroupsFromANRIds(ctx context.Context, eventIds []uuid.UUID) (anrGroups []ANRGroup, err error) { + // Get list of fingerprints and event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id, anr.fingerprint`). + Where(`id in (?)`, eventIds). + Where(`anr.fingerprint != ''`) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + fingerprints := make([]string, 0) + fingerprintSet := make(map[string]struct{}) + eventIdToFingerprint := make(map[uuid.UUID]string) + + for eventDataRows.Next() { + var eventID uuid.UUID + var fingerprint string + if err := eventDataRows.Scan(&eventID, &fingerprint); err != nil { + return nil, err + } + eventIdToFingerprint[eventID] = fingerprint + if _, exists := fingerprintSet[fingerprint]; !exists { + fingerprints = append(fingerprints, fingerprint) + fingerprintSet[fingerprint] = struct{}{} + } + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + // Query groups that match the obtained fingerprints stmt := sqlf.PostgreSQL. From(`public.anr_groups`). Select(`id`). @@ -295,18 +375,34 @@ func GetANRGroupsFromANRIds(ctx context.Context, eventIds []uuid.UUID) (anrGroup Select(`method_name`). Select(`file_name`). Select(`line_number`). - Select(`event_ids`). - // `&&` matches rows by list of uuids - Where(`event_ids && ?`, eventIds) + Select(`fingerprint`). + Where(`fingerprint = ANY(?)`, fingerprints) defer stmt.Close() rows, _ := server.Server.PgPool.Query(ctx, stmt.String(), stmt.Args()...) anrGroups, err = pgx.CollectRows(rows, pgx.RowToStructByNameLax[ANRGroup]) - err = rows.Err() + if err != nil { + return nil, err + } + if rows.Err() != nil { + return nil, rows.Err() + } - return + // Add event ids to obtained ANR groups + fingerprintToGroup := make(map[string]*ANRGroup) + for i := range anrGroups { + fingerprintToGroup[anrGroups[i].Fingerprint] = &anrGroups[i] + } + + for eventID, fingerprint := range eventIdToFingerprint { + if group, ok := fingerprintToGroup[fingerprint]; ok { + group.EventIDs = append(group.EventIDs, eventID) + } + } + + return anrGroups, nil } // PaginateGroups accepts slice of interface GroupID and computes and @@ -369,7 +465,7 @@ func PaginateGroups[T GroupID](groups []T, af *filter.AppFilter) (sliced []T, ne } // NewExceptionGroup constructs a new ExceptionGroup and returns a pointer to it. -func NewExceptionGroup(appId uuid.UUID, exceptionType, message, methodName, fileName string, lineNumber int, fingerprint string, eventIds []uuid.UUID, firstTime time.Time) *ExceptionGroup { +func NewExceptionGroup(appId uuid.UUID, exceptionType, message, methodName, fileName string, lineNumber int, fingerprint string, firstTime time.Time) *ExceptionGroup { return &ExceptionGroup{ AppID: appId, Type: exceptionType, @@ -378,13 +474,12 @@ func NewExceptionGroup(appId uuid.UUID, exceptionType, message, methodName, file FileName: fileName, LineNumber: lineNumber, Fingerprint: fingerprint, - EventIDs: eventIds, FirstEventTime: firstTime, } } // NewANRGroup constructs a new ANRGroup and returns a pointer to it. -func NewANRGroup(appId uuid.UUID, anrType, message, methodName, fileName string, lineNumber int, fingerprint string, eventIds []uuid.UUID, firstTime time.Time) *ANRGroup { +func NewANRGroup(appId uuid.UUID, anrType, message, methodName, fileName string, lineNumber int, fingerprint string, firstTime time.Time) *ANRGroup { return &ANRGroup{ AppID: appId, Type: anrType, @@ -393,7 +488,6 @@ func NewANRGroup(appId uuid.UUID, anrType, message, methodName, fileName string, FileName: fileName, LineNumber: lineNumber, Fingerprint: fingerprint, - EventIDs: eventIds, FirstEventTime: firstTime, } } diff --git a/backend/api/measure/app.go b/backend/api/measure/app.go index 24b2bd05a..a50816285 100644 --- a/backend/api/measure/app.go +++ b/backend/api/measure/app.go @@ -81,8 +81,6 @@ func (a App) GetExceptionGroup(ctx context.Context, id uuid.UUID) (exceptionGrou Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). @@ -106,7 +104,35 @@ func (a App) GetExceptionGroup(ctx context.Context, id uuid.UUID) (exceptionGrou exceptionGroup = &row - return + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`exception.fingerprint = (?)`, exceptionGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + exceptionGroup.EventIDs = eventIds + exceptionGroup.Count = len(eventIds) + + return exceptionGroup, nil } // GetExceptionGroupByFingerprint queries a single exception group by its fingerprint. @@ -121,8 +147,6 @@ func (a App) GetExceptionGroupByFingerprint(ctx context.Context, fingerprint str Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). @@ -145,7 +169,35 @@ func (a App) GetExceptionGroupByFingerprint(ctx context.Context, fingerprint str exceptionGroup = &row - return + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`exception.fingerprint = ?`, exceptionGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + exceptionGroup.EventIDs = eventIds + exceptionGroup.Count = len(eventIds) + + return exceptionGroup, nil } // GetExceptionGroups returns slice of ExceptionGroup @@ -161,18 +213,54 @@ func (a App) GetExceptionGroups(ctx context.Context) (groups []group.ExceptionGr Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). - Where("app_id = ?", a.ID). - OrderBy("count desc") + Where("app_id = ?", a.ID) defer stmt.Close() rows, _ := server.Server.PgPool.Query(ctx, stmt.String(), stmt.Args()...) - return pgx.CollectRows(rows, pgx.RowToStructByNameLax[group.ExceptionGroup]) + exceptionGroups, err := pgx.CollectRows(rows, pgx.RowToStructByNameLax[group.ExceptionGroup]) + + if err != nil { + return nil, err + } + + for i := range exceptionGroups { + + exceptionGroup := &exceptionGroups[i] + + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`exception.fingerprint = ?`, exceptionGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + exceptionGroup.EventIDs = eventIds + exceptionGroup.Count = len(eventIds) + } + + return exceptionGroups, nil } // GetANRGroup queries a single ANR group by its id. @@ -187,8 +275,6 @@ func (a App) GetANRGroup(ctx context.Context, id uuid.UUID) (anrGroup *group.ANR Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). @@ -212,7 +298,35 @@ func (a App) GetANRGroup(ctx context.Context, id uuid.UUID) (anrGroup *group.ANR anrGroup = &row - return + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`anr.fingerprint = ?`, anrGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + anrGroup.EventIDs = eventIds + anrGroup.Count = len(eventIds) + + return anrGroup, nil } // GetANRGroupByFingerprint queries a single ANR group by its fingerprint. @@ -227,8 +341,6 @@ func (a App) GetANRGroupByFingerprint(ctx context.Context, fingerprint string) ( Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). @@ -251,7 +363,35 @@ func (a App) GetANRGroupByFingerprint(ctx context.Context, fingerprint string) ( anrGroup = &row - return + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`anr.fingerprint = ?`, anrGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + anrGroup.EventIDs = eventIds + anrGroup.Count = len(eventIds) + + return anrGroup, nil } // GetANRGroups returns slice of ANRGroup of an app. @@ -266,18 +406,54 @@ func (a App) GetANRGroups(ctx context.Context) (groups []group.ANRGroup, err err Select(`file_name`). Select(`line_number`). Select("fingerprint"). - Select("event_ids"). - Select("COALESCE(array_length(event_ids, 1), 0) as count"). Select("first_event_timestamp"). Select("created_at"). Select("updated_at"). - Where("app_id = ?", a.ID). - OrderBy("count desc") + Where("app_id = ?", a.ID) defer stmt.Close() rows, _ := server.Server.PgPool.Query(ctx, stmt.String(), stmt.Args()...) - return pgx.CollectRows(rows, pgx.RowToStructByNameLax[group.ANRGroup]) + anrGroups, err := pgx.CollectRows(rows, pgx.RowToStructByNameLax[group.ANRGroup]) + + if err != nil { + return nil, err + } + + for i := range anrGroups { + + anrGroup := &anrGroups[i] + + // Get list of event IDs + eventDataStmt := sqlf.From(`default.events`). + Select(`id`). + Where(`anr.fingerprint = ?`, anrGroup.Fingerprint) + + eventDataRows, err := server.Server.ChPool.Query(ctx, eventDataStmt.String(), eventDataStmt.Args()...) + if err != nil { + return nil, err + } + defer eventDataRows.Close() + + var eventIds = []uuid.UUID{} + var eventID uuid.UUID + for eventDataRows.Next() { + if err := eventDataRows.Scan(&eventID); err != nil { + return nil, err + } + + eventIds = append(eventIds, eventID) + } + + if eventDataRows.Err() != nil { + return nil, eventDataRows.Err() + } + + anrGroup.EventIDs = eventIds + anrGroup.Count = len(eventIds) + } + + return anrGroups, nil } // GetSizeMetrics computes app size of the selected app version diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index 2ca982454..f25711b45 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -342,15 +342,16 @@ func (e eventreq) bucketUnhandledExceptions(ctx context.Context, tx *pgx.Tx) (er } if matchedGroup == nil { - exceptionGroup := group.NewExceptionGroup(events[i].AppID, events[i].Exception.GetType(), events[i].Exception.GetMessage(), events[i].Exception.GetMethodName(), events[i].Exception.GetFileName(), events[i].Exception.GetLineNumber(), events[i].Exception.Fingerprint, []uuid.UUID{events[i].ID}, events[i].Timestamp) + exceptionGroup := group.NewExceptionGroup(events[i].AppID, events[i].Exception.GetType(), events[i].Exception.GetMessage(), events[i].Exception.GetMethodName(), events[i].Exception.GetFileName(), events[i].Exception.GetLineNumber(), events[i].Exception.Fingerprint, events[i].Timestamp) if err := exceptionGroup.Insert(ctx, tx); err != nil { return err } - matchedGroup = exceptionGroup + + return nil } if !matchedGroup.EventExists(events[i].ID) { - if err := matchedGroup.AppendEvent(ctx, &events[i], tx); err != nil { + if err := matchedGroup.UpdateTimeStamps(ctx, &events[i], tx); err != nil { return err } } @@ -380,16 +381,16 @@ func (e eventreq) bucketANRs(ctx context.Context, tx *pgx.Tx) (err error) { } if matchedGroup == nil { - anrGroup := group.NewANRGroup(events[i].AppID, events[i].ANR.GetType(), events[i].ANR.GetMessage(), events[i].ANR.GetMethodName(), events[i].ANR.GetFileName(), events[i].ANR.GetLineNumber(), events[i].ANR.Fingerprint, []uuid.UUID{events[i].ID}, events[i].Timestamp) + anrGroup := group.NewANRGroup(events[i].AppID, events[i].ANR.GetType(), events[i].ANR.GetMessage(), events[i].ANR.GetMethodName(), events[i].ANR.GetFileName(), events[i].ANR.GetLineNumber(), events[i].ANR.Fingerprint, events[i].Timestamp) if err := anrGroup.Insert(ctx, tx); err != nil { return err } - matchedGroup = anrGroup + return nil } if !matchedGroup.EventExists(events[i].ID) { - if err := matchedGroup.AppendEvent(ctx, &events[i], tx); err != nil { + if err := matchedGroup.UpdateTimeStamps(ctx, &events[i], tx); err != nil { return err } } diff --git a/backend/cleanup/cleanup/cleanup.go b/backend/cleanup/cleanup/cleanup.go index e76308903..f5fb8a61f 100644 --- a/backend/cleanup/cleanup/cleanup.go +++ b/backend/cleanup/cleanup/cleanup.go @@ -28,8 +28,6 @@ type StaleData struct { AppID string `json:"app_id"` RetentionDate time.Time `json:"retention_date"` EventIDs []string `json:"event_ids"` - ANRGroupIDs []string `json:"anr_group_ids"` - CrashGroupIds []string `json:"crash_group_ids"` Attachments []Attachment `json:"attachments"` } @@ -43,58 +41,6 @@ func DeleteStaleData(ctx context.Context) { for _, st := range staleData { - // Update ANR groups in postgres - if len(st.ANRGroupIDs) > 0 { - fmt.Printf("Deleting events from %v ANR groups for app_id: %v\n", len(st.ANRGroupIDs), st.AppID) - - updateANRGroupsStmt := ` - UPDATE public.anr_groups - SET event_ids = ( - SELECT array( - SELECT unnest(event_ids) - EXCEPT - SELECT unnest($1::uuid[]) - ) - ) - WHERE app_id = $2 - AND event_ids && $3::uuid[] - AND id = ANY($4::uuid[]); - ` - - if _, err := server.Server.PgPool.Exec(ctx, updateANRGroupsStmt, st.EventIDs, st.AppID, st.EventIDs, st.ANRGroupIDs); err != nil { - fmt.Printf("Failed to delete events from %v ANR groups for app_id: %v, err: %v\n", len(st.ANRGroupIDs), st.AppID, err) - return - } - - fmt.Printf("Deleted events from %v ANR groups for app_id: %v\n", len(st.ANRGroupIDs), st.AppID) - } - - // Update Crash groups in postgres - if len(st.CrashGroupIds) > 0 { - fmt.Printf("Deleting events from %v crash groups for app_id: %v\n", len(st.CrashGroupIds), st.AppID) - - updateCrashGroupsStmt := ` - UPDATE public.unhandled_exception_groups - SET event_ids = ( - SELECT array( - SELECT unnest(event_ids) - EXCEPT - SELECT unnest($1::uuid[]) - ) - ) - WHERE app_id = $2 - AND event_ids && $3::uuid[] - AND id = ANY($4::uuid[]); - ` - - if _, err := server.Server.PgPool.Exec(ctx, updateCrashGroupsStmt, st.EventIDs, st.AppID, st.EventIDs, st.CrashGroupIds); err != nil { - fmt.Printf("Failed to delete events from %v crash groups for app_id: %v, err: %v\n", len(st.CrashGroupIds), st.AppID, err) - return - } - - fmt.Printf("Deleted events from %v crash groups for app_id: %v\n", len(st.CrashGroupIds), st.AppID) - } - // Delete attachments from object storage if len(st.Attachments) > 0 { fmt.Printf("Deleting %v attachments for app_id: %v\n", len(st.Attachments), st.AppID) @@ -196,60 +142,10 @@ func fetchStaleData(ctx context.Context) ([]StaleData, error) { staleEventIDs = append(staleEventIDs, eventID) } - // Fetch ANR group IDs that contain these events - var anrGroupIDs []string - if len(staleEventIDs) > 0 { - fetchANRGroupsStmt := sqlf.Select("id"). - From("public.anr_groups"). - Where("app_id = ?", appID). - Where("event_ids && ?", staleEventIDs) - - anrGroupRows, err := server.Server.PgPool.Query(ctx, fetchANRGroupsStmt.String(), fetchANRGroupsStmt.Args()...) - if err != nil { - fmt.Printf("Failed to fetch ANR groups: %v\n", err) - continue - } - - var anrGroupID string - for anrGroupRows.Next() { - if err := anrGroupRows.Scan(&anrGroupID); err != nil { - fmt.Printf("Failed to scan ANR group ID: %v\n", err) - continue - } - anrGroupIDs = append(anrGroupIDs, anrGroupID) - } - } - - // Fetch unhandled exception group IDs that contain these events - var crashGroupIDs []string - if len(staleEventIDs) > 0 { - fetchCrashGroupsStmt := sqlf.Select("id"). - From("public.unhandled_exception_groups"). - Where("app_id = ?", appID). - Where("event_ids && ?", staleEventIDs) - - ueGroupRows, err := server.Server.PgPool.Query(ctx, fetchCrashGroupsStmt.String(), fetchCrashGroupsStmt.Args()...) - if err != nil { - fmt.Printf("Failed to fetch unhandled exception groups: %v\n", err) - continue - } - - var ueGroupID string - for ueGroupRows.Next() { - if err := ueGroupRows.Scan(&ueGroupID); err != nil { - fmt.Printf("Failed to scan unhandled exception group ID: %v\n", err) - continue - } - crashGroupIDs = append(crashGroupIDs, ueGroupID) - } - } - staleData = append(staleData, StaleData{ AppID: appID, RetentionDate: retentionDate, EventIDs: staleEventIDs, - ANRGroupIDs: anrGroupIDs, - CrashGroupIds: crashGroupIDs, Attachments: staleAttachments, }) } diff --git a/self-host/postgres/20231228033348_create_unhandled_exception_groups_table.sql b/self-host/postgres/20231228033348_create_unhandled_exception_groups_table.sql index b9a8952bf..4e7b0d5bc 100644 --- a/self-host/postgres/20231228033348_create_unhandled_exception_groups_table.sql +++ b/self-host/postgres/20231228033348_create_unhandled_exception_groups_table.sql @@ -8,7 +8,6 @@ create table if not exists public.unhandled_exception_groups ( file_name text not null, line_number int not null, fingerprint varchar(32) not null, - event_ids uuid[] not null, first_event_timestamp timestamptz not null, created_at timestamptz not null default now(), updated_at timestamptz not null default now() @@ -22,7 +21,6 @@ comment on column public.unhandled_exception_groups.method_name is 'method name comment on column public.unhandled_exception_groups.file_name is 'file name where the exception occured'; comment on column public.unhandled_exception_groups.line_number is 'line number where the exception occured'; comment on column public.unhandled_exception_groups.fingerprint is 'fingerprint of the unhandled exception'; -comment on column public.unhandled_exception_groups.event_ids is 'list of associated event ids'; comment on column public.unhandled_exception_groups.first_event_timestamp is 'utc timestamp of the oldest event in the group'; comment on column public.unhandled_exception_groups.created_at is 'utc timestamp at the time of record creation'; comment on column public.unhandled_exception_groups.updated_at is 'utc timestamp at the time of record updation'; diff --git a/self-host/postgres/20231228044339_create_anr_groups_table.sql b/self-host/postgres/20231228044339_create_anr_groups_table.sql index 76e89659e..cbae6b785 100644 --- a/self-host/postgres/20231228044339_create_anr_groups_table.sql +++ b/self-host/postgres/20231228044339_create_anr_groups_table.sql @@ -8,7 +8,6 @@ create table if not exists public.anr_groups ( file_name text not null, line_number int not null, fingerprint varchar(32) not null, - event_ids uuid[] not null, first_event_timestamp timestamptz not null, created_at timestamptz not null default now(), updated_at timestamptz not null default now() @@ -22,7 +21,6 @@ comment on column public.anr_groups.method_name is 'method name where the anr oc comment on column public.anr_groups.file_name is 'file name where the anr occured'; comment on column public.anr_groups.line_number is 'line number where the anr occured'; comment on column public.anr_groups.fingerprint is 'fingerprint of the anr'; -comment on column public.anr_groups.event_ids is 'list of associated event ids'; comment on column public.anr_groups.first_event_timestamp is 'utc timestamp of the oldest event in the group'; comment on column public.anr_groups.created_at is 'utc timestamp at the time of record creation'; comment on column public.anr_groups.updated_at is 'utc timestamp at the time of record updation';