Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[GCAL] Enable notifications and reminders when a superuser token is not supported #272

Merged
merged 41 commits into from
Aug 1, 2023
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
4d3a2bb
nil control
fmartingr Jul 25, 2023
fe6fe75
use subscriptions default ttl
fmartingr Jul 25, 2023
7634f2c
return specific error when no superuser token can be used
fmartingr Jul 25, 2023
9dc21a2
notification formatting logic to other file
fmartingr Jul 25, 2023
a436769
Client.GetEventsBetweenDates
fmartingr Jul 25, 2023
18767a3
typo: import
fmartingr Jul 25, 2023
8de4223
shorter command name :)
fmartingr Jul 25, 2023
af611a9
availability without super user token
fmartingr Jul 25, 2023
24c2c64
delete store subs only if we successfuly do on remote
fmartingr Jul 25, 2023
0ecad3f
comment
fmartingr Jul 25, 2023
165948c
ignore missing subs
fmartingr Jul 25, 2023
2e6c571
update sync method
fmartingr Jul 25, 2023
cc88b2e
allow switching acting user on the fly
fmartingr Jul 25, 2023
0def44a
daily summary without super user token
fmartingr Jul 25, 2023
c5f0e1c
formatted imports
fmartingr Jul 25, 2023
0b7bce2
calendar -> calendarEvents
fmartingr Jul 25, 2023
100497a
processAllDailySummaryByUser
fmartingr Jul 25, 2023
d90fb5e
revert silenced error under notifications
fmartingr Jul 25, 2023
4345953
msgraph GetEventsBetweenDates
fmartingr Jul 25, 2023
57b30f9
Update server/mscalendar/daily_summary.go
fmartingr Jul 26, 2023
92de736
[GCAL] Create event logic (#269)
fmartingr Jul 25, 2023
3e3f43d
Client.GetEventsBetweenDates
fmartingr Jul 25, 2023
5c05a74
fix: store last post time
fmartingr Jul 26, 2023
5dbc701
refactored availability logic
fmartingr Jul 26, 2023
19abc70
availability test
fmartingr Jul 26, 2023
c3336e9
refactored daily summary logic
fmartingr Jul 26, 2023
e4f73fd
test: availability
fmartingr Jul 26, 2023
9951416
daily summary tests (wip)
fmartingr Jul 26, 2023
a92d545
daily summary test
fmartingr Jul 27, 2023
c090410
fix: merge duplications
fmartingr Jul 27, 2023
ecca739
slack attachment for notifications
fmartingr Jul 27, 2023
6f2b1d7
lint and test
fmartingr Jul 27, 2023
351730c
goimports
fmartingr Jul 27, 2023
7c3b63d
Update server/mscalendar/availability.go
fmartingr Jul 31, 2023
d8745f6
remove logges
fmartingr Jul 31, 2023
58e7b9f
nullify client when changing acting user
fmartingr Jul 31, 2023
9aac20f
move withActiveUser filter to fenced code outside of general method
fmartingr Jul 31, 2023
c9378ce
tests: handling more scenarios
fmartingr Jul 31, 2023
b139ac5
Fixed test after bugfix
fmartingr Jul 31, 2023
08b039e
fixed test assertions
fmartingr Jul 31, 2023
b240b85
allow engine copy to perform safe state mutations
fmartingr Jul 31, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 7 additions & 11 deletions server/mscalendar/availability.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package mscalendar

import (
"fmt"
"log"
"time"

"github.com/mattermost/mattermost-server/v6/model"
Expand Down Expand Up @@ -118,6 +117,12 @@ func (m *mscalendar) retrieveUsersToSync(userIndex store.UserIndex, syncJobSumma
users = append(users, user)

if fetchIndividually {
err = m.Filter(withActingUser(user.MattermostUserID))
if err != nil {
m.Logger.Warnf("Not able to enable active user %s from user index. err=%v", user.MattermostUserID, err)
continue
}

calendarUser := newUserFromStoredUser(user)
calendarEvents, err := m.GetCalendarEvents(calendarUser, start, end)
if err != nil {
Expand Down Expand Up @@ -442,20 +447,11 @@ func (m *mscalendar) setStatusOrAskUser(user *store.User, currentStatus *model.S
}

func (m *mscalendar) GetCalendarEvents(user *User, start, end time.Time) (*remote.ViewCalendarResponse, error) {
err := m.Filter(withActingUser(user.MattermostUserID))
if err != nil {
return nil, errors.Wrap(err, "error withActingUser")
}

err = m.Filter(withClient)
err := m.Filter(withClient)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you see an opportunity to make this code less error-prone? It concerns me that it's easy to make an error based on where this call is placed

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That was the intention when I removed the withActingUser from this method. Now we can call this one without problems from other places since the new withActingUser only gets called from the fenced code in the "main" calls.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, but what can we do to avoid needing to have withActingUser in the fenced code? Just want to make sure we don't run into something similar later, especially if it's something not caught by a unit test

if err != nil {
return nil, errors.Wrap(err, "errror withClient")
}

log.Println(m.client)
log.Println(user)
log.Println(user.Remote)

events, err := m.client.GetEventsBetweenDates(user.Remote.ID, start, end)
if err != nil {
return nil, errors.Wrapf(err, "error getting events for user %s", user.MattermostUserID)
Expand Down
101 changes: 100 additions & 1 deletion server/mscalendar/availability_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ func TestRetrieveUsersToSyncIndividually(t *testing.T) {
c, r, _, s, papi := client.(*mock_remote.MockClient), e.Remote.(*mock_remote.MockRemote), e.Poster.(*mock_bot.MockPoster), e.Store.(*mock_store.MockStore), e.PluginAPI.(*mock_plugin_api.MockPluginAPI)
s.EXPECT().LoadUser(testUser.MattermostUserID).Return(testUser, nil).Times(2)

events := []*remote.Event{newTestEvent("", "test")}
events := []*remote.Event{newTestEvent("1", "", "test")}
papi.EXPECT().GetMattermostUser(testUser.MattermostUserID)
r.EXPECT().MakeClient(gomock.Any(), testUser.OAuth2Token).Return(client)
c.EXPECT().GetEventsBetweenDates(testUser.Remote.ID, gomock.Any(), gomock.Any()).Return(events, nil)
Expand All @@ -430,6 +430,105 @@ func TestRetrieveUsersToSyncIndividually(t *testing.T) {
Events: events,
}})
})

t.Run("one user should be synced, one user shouldn't", func(t *testing.T) {
testUser := newTestUser()
testUser.Settings.UpdateStatus = true
testUser.Settings.ReceiveReminders = true

testUser2 := newTestUserNumbered(1)

userIndex := []*store.UserShort{
{
MattermostUserID: testUser.MattermostUserID,
RemoteID: testUser.Remote.ID,
Email: testUser.Remote.Mail,
},
{
MattermostUserID: testUser2.MattermostUserID,
RemoteID: testUser2.Remote.ID,
Email: testUser2.Remote.Mail,
},
}
ctrl := gomock.NewController(t)
defer ctrl.Finish()

e, client := makeStatusSyncTestEnv(ctrl)

c, r, _, s, papi := client.(*mock_remote.MockClient), e.Remote.(*mock_remote.MockRemote), e.Poster.(*mock_bot.MockPoster), e.Store.(*mock_store.MockStore), e.PluginAPI.(*mock_plugin_api.MockPluginAPI)
s.EXPECT().LoadUser(testUser.MattermostUserID).Return(testUser, nil).Times(2)
s.EXPECT().LoadUser(testUser2.MattermostUserID).Return(testUser2, nil)

events := []*remote.Event{newTestEvent("1", "", "test")}
papi.EXPECT().GetMattermostUser(testUser.MattermostUserID)
r.EXPECT().MakeClient(gomock.Any(), testUser.OAuth2Token).Return(client)
c.EXPECT().GetEventsBetweenDates(testUser.Remote.ID, gomock.Any(), gomock.Any()).Return(events, nil)

m := New(e, "").(*mscalendar)
jobSummary := &StatusSyncJobSummary{}

users, responses, err := m.retrieveUsersToSync(userIndex, jobSummary, true)
require.NoError(t, err)
require.Equal(t, users, []*store.User{testUser})
require.Equal(t, responses, []*remote.ViewCalendarResponse{{
RemoteUserID: testUser.Remote.ID,
Events: events,
}})
})

t.Run("two users should be synced", func(t *testing.T) {
testUser := newTestUserNumbered(1)
testUser.Settings.UpdateStatus = true
testUser.Settings.ReceiveReminders = true

testUser2 := newTestUserNumbered(2)
testUser2.Settings.UpdateStatus = true
testUser2.Settings.ReceiveReminders = true

userIndex := []*store.UserShort{
{
MattermostUserID: testUser.MattermostUserID,
RemoteID: testUser.Remote.ID,
Email: testUser.Remote.Mail,
},
{
MattermostUserID: testUser2.MattermostUserID,
RemoteID: testUser2.Remote.ID,
Email: testUser2.Remote.Mail,
},
}
ctrl := gomock.NewController(t)
defer ctrl.Finish()

e, client := makeStatusSyncTestEnv(ctrl)

c, r, _, s, papi := client.(*mock_remote.MockClient), e.Remote.(*mock_remote.MockRemote), e.Poster.(*mock_bot.MockPoster), e.Store.(*mock_store.MockStore), e.PluginAPI.(*mock_plugin_api.MockPluginAPI)
s.EXPECT().LoadUser(testUser.MattermostUserID).Return(testUser, nil).Times(2)
s.EXPECT().LoadUser(testUser2.MattermostUserID).Return(testUser2, nil).Times(2)

eventsUser1 := []*remote.Event{newTestEvent("1", "", "test")}
eventsUser2 := []*remote.Event{newTestEvent("2", "", "test2")}
papi.EXPECT().GetMattermostUser(testUser.MattermostUserID)
papi.EXPECT().GetMattermostUser(testUser2.MattermostUserID)
r.EXPECT().MakeClient(gomock.Any(), testUser.OAuth2Token).Return(client)
r.EXPECT().MakeClient(gomock.Any(), testUser2.OAuth2Token).Return(client)
c.EXPECT().GetEventsBetweenDates(testUser.Remote.ID, gomock.Any(), gomock.Any()).Return(eventsUser1, nil)
c.EXPECT().GetEventsBetweenDates(testUser2.Remote.ID, gomock.Any(), gomock.Any()).Return(eventsUser2, nil)

m := New(e, "").(*mscalendar)
jobSummary := &StatusSyncJobSummary{}

users, responses, err := m.retrieveUsersToSync(userIndex, jobSummary, true)
require.NoError(t, err)
require.Equal(t, users, []*store.User{testUser, testUser2})
require.Equal(t, responses, []*remote.ViewCalendarResponse{{
fmartingr marked this conversation as resolved.
Show resolved Hide resolved
RemoteUserID: testUser.Remote.ID,
Events: eventsUser1,
}, {
RemoteUserID: testUser2.Remote.ID,
Events: eventsUser2,
}})
})
}

func makeStatusSyncTestEnv(ctrl *gomock.Controller) (Env, remote.Client) {
Expand Down
6 changes: 3 additions & 3 deletions server/mscalendar/daily_summary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ Wednesday February 12, 2020
LastPostTime: "",
},
},
}, nil).Times(2)
}, nil).Times(3)

s.EXPECT().LoadUser("user3_mm_id").Return(&store.User{
MattermostUserID: "user3_mm_id",
Expand All @@ -231,7 +231,7 @@ Wednesday February 12, 2020
}, nil)

papi.EXPECT().GetMattermostUser("user1_mm_id").Times(2)
papi.EXPECT().GetMattermostUser("user2_mm_id").Times(1)
papi.EXPECT().GetMattermostUser("user2_mm_id").Times(2)

mockClient.EXPECT().GetMailboxSettings("user1_remote_id").Return(&remote.MailboxSettings{
TimeZone: "Eastern Standard Time",
Expand All @@ -240,7 +240,7 @@ Wednesday February 12, 2020
TimeZone: "Pacific Standard Time",
}, nil)

mockRemote.EXPECT().MakeClient(context.TODO(), gomock.Any()).Return(mockClient)
mockRemote.EXPECT().MakeClient(context.TODO(), gomock.Any()).Return(mockClient).Times(2)

mockClient.EXPECT().GetDefaultCalendarView("user1_remote_id", gomock.Any(), gomock.Any()).Return([]*remote.Event{}, nil)
mockClient.EXPECT().GetDefaultCalendarView("user2_remote_id", gomock.Any(), gomock.Any()).Return([]*remote.Event{
Expand Down
1 change: 1 addition & 0 deletions server/mscalendar/filters.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func withRemoteUser(user *User) func(m *mscalendar) error {
func withActingUser(mattermostUserID string) func(m *mscalendar) error {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hope this is not cheating 🙃

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not cheating per se, but maybe error-prone. This is being called in a service account context, so we are changing the state of m temporarily while handling a given user's events. This makes it difficult to do concurrent operations across users.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the only way I imagined of leaving the architecture as it is, without moving other parts of the logic outside of the main "calendar" logic, but I do not know how this could affect elsewhere, if anything. Only running under GCal at the moment. Open to suggestions!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In general this seems fine, though just concerned about potential pitfalls of mutating the state. Really that's just an artifact of the .Filter() method in general. It's mainly meant to be used in a per-user context, but we introduced the superuser token later. And now we're doing sort of a hybrid, where we're using user-level OAuth tokens in a scheduled job context.

I think it's fine how this PR implements it, though we will need to rethink this if we want to perform these requests concurrently across different users. Speaking of which, what are your thoughts on the performance of the current approach, versus fanning out the requests in some fashion? The timing of reminders etc. depends on the speed of this operation, so I'm thinking we'll need some performance optimization here in the case of sending individual requests for each user.

Copy link
Contributor Author

@fmartingr fmartingr Jul 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In order to parallelize work we can do a Filter function that returns a copy of the mscalendar with the context switched so we can send out all the jobs to goroutines and wait for all of them to finish using a waitgroup. Is not the best approach, but it surely helps.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good to me 👍

return func(m *mscalendar) error {
m.actingUser = NewUser(mattermostUserID)
m.client = nil
return nil
}
}
Expand Down
23 changes: 14 additions & 9 deletions server/mscalendar/notification_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package mscalendar

import (
"context"
"fmt"
"testing"

"github.com/stretchr/testify/require"
Expand All @@ -29,10 +30,10 @@ func newTestNotificationProcessor(env Env) NotificationProcessor {
return processor
}

func newTestEvent(locationDisplayName string, subjectDisplayName string) *remote.Event {
func newTestEvent(identifier, locationDisplayName string, subjectDisplayName string) *remote.Event {
return &remote.Event{
ID: "remote_event_id",
ICalUID: "remote_event_uid",
ID: fmt.Sprintf("remote_event_id_%s", identifier),
ICalUID: fmt.Sprintf("remote_event_uid_%s", identifier),
Organizer: &remote.Attendee{
EmailAddress: &remote.EmailAddress{
Address: "event_organizer_email",
Expand Down Expand Up @@ -65,15 +66,19 @@ func newTestSubscription() *store.Subscription {
}

func newTestUser() *store.User {
return newTestUserNumbered(1)
}

func newTestUserNumbered(number int) *store.User {
return &store.User{
Settings: store.Settings{
EventSubscriptionID: "remote_subscription_id",
EventSubscriptionID: fmt.Sprintf("remote_subscription_id_%d", number),
},
Remote: &remote.User{ID: "remote_user_id"},
Remote: &remote.User{ID: fmt.Sprintf("remote_user_id_%d", number)},
OAuth2Token: &oauth2.Token{
AccessToken: "creator_oauth_token",
AccessToken: fmt.Sprintf("creator_oauth_token_%d", number),
},
MattermostUserID: "creator_mm_id",
MattermostUserID: fmt.Sprintf("creator_mm_id_%d", number),
}
}

Expand All @@ -83,7 +88,7 @@ func newTestNotification(clientState string, recommendRenew bool) *remote.Notifi
SubscriptionID: "remote_subscription_id",
IsBare: true,
SubscriptionCreator: &remote.User{},
Event: newTestEvent("event_location_display_name", "event_subject"),
Event: newTestEvent("1", "event_location_display_name", "event_subject"),
Subscription: &remote.Subscription{},
ClientState: clientState,
RecommendRenew: recommendRenew,
Expand Down Expand Up @@ -114,7 +119,7 @@ func TestProcessNotification(t *testing.T) {
name: "prior event exists",
expectedError: "",
notification: newTestNotification("stored_client_state", false),
priorEvent: newTestEvent("prior_event_location_display_name", "other_event_subject"),
priorEvent: newTestEvent("1", "prior_event_location_display_name", "other_event_subject"),
}, {
name: "sub renewal recommended",
expectedError: "",
Expand Down