From 323051ef2ebc42ff39204cecdd99b96a8314487f Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:23:37 +0700 Subject: [PATCH 01/11] TW-2165: Add store members function --- lib/src/database/database_api.dart | 2 ++ lib/src/database/hive_collections_database.dart | 9 +++++++++ lib/src/database/hive_database.dart | 9 +++++++++ 3 files changed, 20 insertions(+) diff --git a/lib/src/database/database_api.dart b/lib/src/database/database_api.dart index 2d559426d..2a14b3078 100644 --- a/lib/src/database/database_api.dart +++ b/lib/src/database/database_api.dart @@ -82,6 +82,8 @@ abstract class DatabaseApi { Future> getUsers(Room room); + Future storeUsers(List users, Room room); + Future> getEventList( Room room, { int start = 0, diff --git a/lib/src/database/hive_collections_database.dart b/lib/src/database/hive_collections_database.dart index 2004fa5e3..b19cabf78 100644 --- a/lib/src/database/hive_collections_database.dart +++ b/lib/src/database/hive_collections_database.dart @@ -812,6 +812,15 @@ class HiveCollectionsDatabase extends DatabaseApi { return users; } + @override + Future storeUsers(List users, Room room) async { + for (final user in users) { + final key = TupleKey(room.id, user.id).toString(); + await _roomMembersBox.put(key, user.toJson()); + } + return; + } + @override Future insertClient( String name, diff --git a/lib/src/database/hive_database.dart b/lib/src/database/hive_database.dart index 47277f291..e7d96286a 100644 --- a/lib/src/database/hive_database.dart +++ b/lib/src/database/hive_database.dart @@ -745,6 +745,15 @@ class FamedlySdkHiveDatabase extends DatabaseApi { return users; } + @override + Future storeUsers(List users, Room room) async { + for (final user in users) { + final key = MultiKey(room.id, user.id).toString(); + await _roomMembersBox.put(key, user.toJson()); + } + return Future.value(); + } + @override Future insertClient( String name, From 14e7384a239e99685fc1b23e4bc1e696333609c4 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:23:44 +0700 Subject: [PATCH 02/11] TW-2165: Add test for store members function --- test/database_api_test.dart | 47 ++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/test/database_api_test.dart b/test/database_api_test.dart index ecdec026d..ae38f0b99 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -81,20 +81,20 @@ void testDatabase( final toDeviceQueue = await database.getToDeviceEventQueue(); expect(toDeviceQueue.isEmpty, true); }); - test('storeFile', () async { - await database.storeFile( - Uri.parse('mxc://test'), Uint8List.fromList([0]), 0); - final file = await database.getFile(Uri.parse('mxc://test')); - expect(file != null, database.supportsFileStoring); - }); - test('getFile', () async { - await database.getFile(Uri.parse('mxc://test')); - }); - test('deleteOldFiles', () async { - await database.deleteOldFiles(1); - final file = await database.getFile(Uri.parse('mxc://test')); - expect(file == null, true); - }); + // test('storeFile', () async { + // await database.storeFile( + // Uri.parse('mxc://test'), Uint8List.fromList([0]), 0); + // final file = await database.getFile(Uri.parse('mxc://test')); + // expect(file != null, database.supportsFileStoring); + // }); + // test('getFile', () async { + // await database.getFile(Uri.parse('mxc://test')); + // }); + // test('deleteOldFiles', () async { + // await database.deleteOldFiles(1); + // final file = await database.getFile(Uri.parse('mxc://test')); + // expect(file == null, true); + // }); test('storeRoomUpdate', () async { final roomUpdate = JoinedRoomUpdate.fromJson({ 'highlight_count': 0, @@ -230,6 +230,25 @@ void testDatabase( Room(id: '!testroom:example.com', client: Client('testclient'))); expect(users.isEmpty, true); }); + test('storeUsers', () async { + final room = + Room(id: '!testroom:example.com', client: Client('testclient')); + await database.storeUsers( + [ + User( + '@bob:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ) + ], + Room(id: '!testroom:example.com', client: Client('testclient')), + ); + final users = await database.getUsers( + Room(id: '!testroom:example.com', client: Client('testclient')), + ); + expect(users.single.id, '@bob:example.org'); + }); test('removeEvent', () async { await database.removeEvent('\$event:example.com', '!testroom:example.com'); final event = await database.getEventById('\$event:example.com', From c2cfea5b01917257177e4e2881464078338d1cf1 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:24:15 +0700 Subject: [PATCH 03/11] TW-2165: Improve get/store Participants --- lib/src/room.dart | 38 +++++++++++++++++++++++++------------ test/database_api_test.dart | 28 +++++++++++++-------------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 9aefeb708..de3694566 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1363,10 +1363,14 @@ class Room { /// [[Membership.join, Membership.invite, Membership.knock]] /// Set [cache] to `false` if you do not want to cache the users in memory /// for this session which is highly recommended for large public rooms. - Future> requestParticipants( - [List membershipFilter = displayMembershipsFilter, - bool suppressWarning = false, - bool cache = true]) async { + Future> requestParticipants([ + List membershipFilter = displayMembershipsFilter, + bool suppressWarning = false, + bool cache = true, + String? at, + Membership? membership, + Membership? notMembership, + ]) async { if (!participantListComplete && partial) { // we aren't fully loaded, maybe the users are in the database final users = await client.database?.getUsers(this) ?? []; @@ -1385,13 +1389,20 @@ class Room { membershipFilter, suppressWarning, cache, + at, + membership, + notMembership, ); } - Future> requestParticipantsFromServer( - [List membershipFilter = displayMembershipsFilter, - bool suppressWarning = false, - bool cache = true]) async { + Future> requestParticipantsFromServer([ + List membershipFilter = displayMembershipsFilter, + bool suppressWarning = false, + bool cache = true, + String? at, + Membership? membership, + Membership? notMembership, + ]) async { final memberCount = summary.mJoinedMemberCount; if (!suppressWarning && cache && memberCount != null && memberCount > 100) { Logs().w(''' @@ -1401,16 +1412,19 @@ class Room { '''); } - final matrixEvents = await client.getMembersByRoom(id); + final matrixEvents = await client.getMembersByRoom( + id, + at: at, + membership: membership, + notMembership: notMembership, + ); final users = matrixEvents ?.map((e) => Event.fromMatrixEvent(e, this).asUser) .toList() ?? []; if (cache) { - for (final user in users) { - setState(user); // at *least* cache this in-memory - } + await client.database?.storeUsers(users, this); } _requestedParticipants = cache; diff --git a/test/database_api_test.dart b/test/database_api_test.dart index ae38f0b99..78b0430b8 100644 --- a/test/database_api_test.dart +++ b/test/database_api_test.dart @@ -81,20 +81,20 @@ void testDatabase( final toDeviceQueue = await database.getToDeviceEventQueue(); expect(toDeviceQueue.isEmpty, true); }); - // test('storeFile', () async { - // await database.storeFile( - // Uri.parse('mxc://test'), Uint8List.fromList([0]), 0); - // final file = await database.getFile(Uri.parse('mxc://test')); - // expect(file != null, database.supportsFileStoring); - // }); - // test('getFile', () async { - // await database.getFile(Uri.parse('mxc://test')); - // }); - // test('deleteOldFiles', () async { - // await database.deleteOldFiles(1); - // final file = await database.getFile(Uri.parse('mxc://test')); - // expect(file == null, true); - // }); + test('storeFile', () async { + await database.storeFile( + Uri.parse('mxc://test'), Uint8List.fromList([0]), 0); + final file = await database.getFile(Uri.parse('mxc://test')); + expect(file != null, database.supportsFileStoring); + }); + test('getFile', () async { + await database.getFile(Uri.parse('mxc://test')); + }); + test('deleteOldFiles', () async { + await database.deleteOldFiles(1); + final file = await database.getFile(Uri.parse('mxc://test')); + expect(file == null, true); + }); test('storeRoomUpdate', () async { final roomUpdate = JoinedRoomUpdate.fromJson({ 'highlight_count': 0, From 4f219c0aa3f1202e13d30b51e8e30e5f37161be5 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:30:11 +0700 Subject: [PATCH 04/11] fixup! TW-2165: Improve get/store Participants --- lib/src/room.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index de3694566..d71e63527 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1395,14 +1395,14 @@ class Room { ); } - Future> requestParticipantsFromServer([ + Future> requestParticipantsFromServer({ List membershipFilter = displayMembershipsFilter, bool suppressWarning = false, bool cache = true, String? at, Membership? membership, Membership? notMembership, - ]) async { + }) async { final memberCount = summary.mJoinedMemberCount; if (!suppressWarning && cache && memberCount != null && memberCount > 100) { Logs().w(''' From 77ab2d8686142ee66d5470dee99ae40c39f48592 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:35:22 +0700 Subject: [PATCH 05/11] fixup! fixup! TW-2165: Improve get/store Participants --- lib/src/room.dart | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index d71e63527..9a19633bd 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1337,12 +1337,13 @@ class Room { /// List `membershipFilter` defines with what membership do you want the /// participants, default set to /// [[Membership.join, Membership.invite, Membership.knock]] - List getParticipants( - [List membershipFilter = const [ - Membership.join, - Membership.invite, - Membership.knock, - ]]) { + List getParticipants({ + List membershipFilter = const [ + Membership.join, + Membership.invite, + Membership.knock, + ], + }) { final members = states[EventTypes.RoomMember]; if (members != null) { return members.entries @@ -1363,14 +1364,14 @@ class Room { /// [[Membership.join, Membership.invite, Membership.knock]] /// Set [cache] to `false` if you do not want to cache the users in memory /// for this session which is highly recommended for large public rooms. - Future> requestParticipants([ + Future> requestParticipants({ List membershipFilter = displayMembershipsFilter, bool suppressWarning = false, bool cache = true, String? at, Membership? membership, Membership? notMembership, - ]) async { + }) async { if (!participantListComplete && partial) { // we aren't fully loaded, maybe the users are in the database final users = await client.database?.getUsers(this) ?? []; @@ -1382,16 +1383,16 @@ class Room { // Do not request users from the server if we have already done it // in this session or have a complete list locally. if (_requestedParticipants || participantListComplete) { - return getParticipants(membershipFilter); + return getParticipants(membershipFilter: membershipFilter); } return await requestParticipantsFromServer( - membershipFilter, - suppressWarning, - cache, - at, - membership, - notMembership, + membershipFilter: membershipFilter, + suppressWarning: suppressWarning, + cache: cache, + at: at, + membership: membership, + notMembership: notMembership, ); } From 23a5c3091c3f43f1b68fe623a7f768f00b0926b1 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 20 Dec 2024 11:40:04 +0700 Subject: [PATCH 06/11] fixup! fixup! fixup! TW-2165: Improve get/store Participants --- lib/src/utils/pushrule_evaluator.dart | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/src/utils/pushrule_evaluator.dart b/lib/src/utils/pushrule_evaluator.dart index a531325fe..37f7436de 100644 --- a/lib/src/utils/pushrule_evaluator.dart +++ b/lib/src/utils/pushrule_evaluator.dart @@ -299,7 +299,11 @@ class PushruleEvaluator { } EvaluatedPushRuleAction match(Event event) { - final memberCount = event.room.getParticipants([Membership.join]).length; + final memberCount = event.room.getParticipants( + membershipFilter: [ + Membership.join, + ], + ).length; final displayName = event.room .unsafeGetUserFromMemoryOrFallback(event.room.client.userID!) .displayName; From 2db7f0bc025487f8c88bf5f9a7dd41a0d1c9fc98 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Mon, 23 Dec 2024 23:18:46 +0700 Subject: [PATCH 07/11] fixup! fixup! fixup! fixup! TW-2165: Improve get/store Participants --- lib/src/room.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 9a19633bd..bd4992f02 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1372,7 +1372,7 @@ class Room { Membership? membership, Membership? notMembership, }) async { - if (!participantListComplete && partial) { + if (!participantListComplete || partial) { // we aren't fully loaded, maybe the users are in the database final users = await client.database?.getUsers(this) ?? []; for (final user in users) { From 42a24a529456ae5ae98c26b7264919aa98bb5288 Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Mon, 23 Dec 2024 23:27:25 +0700 Subject: [PATCH 08/11] TW-2165: Add adr for this ticket --- doc/adr/0005-support-store-members-in-hive.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 doc/adr/0005-support-store-members-in-hive.md diff --git a/doc/adr/0005-support-store-members-in-hive.md b/doc/adr/0005-support-store-members-in-hive.md new file mode 100644 index 000000000..c7e16ef0e --- /dev/null +++ b/doc/adr/0005-support-store-members-in-hive.md @@ -0,0 +1,45 @@ +# 5. Support store members in hive + +Date: 2024-12-23 + +## Status + +Accepted + +- Issue: [#2165](https://github.com/linagora/twake-on-matrix/issues/2165) + +## Context + +- Not all of the members are displayed in the drop-down list +- The members only store in the memory, so when the user refreshes the page, the members are lost. + +## Decision + +- Store the members in the hive to keep the members when the user refreshes the page. +- Add some properties to request the members from the server. + +```dart + +Future> requestParticipantsFromServer({ + List membershipFilter = displayMembershipsFilter, + bool suppressWarning = false, + bool cache = true, + String? at, + Membership? membership, + Membership? notMembership, + }) {} + +``` + +- `at`: The point in time (pagination token) to return members for in the room. +This token can be obtained from a prev_batch token returned for each room by the sync API. +Defaults to the current state of the room, as determined by the server. + +- `membership`: The kind of membership to filter for. Defaults to no filtering if unspecified. +When specified alongside `not_membership`, the two parameters create an ‘or’ condition: either the membership is the same as `membership` or is not the same as `not_membership`. + +- `notMembership`: The kind of membership to exclude from the results. Defaults to no filtering if unspecified. + +## Consequences + +- The members are stored in the hive, so the members are not lost when the user refreshes the page. From 14fd0ce7745f4e4fdb3d1c982e76a6fb7c12819e Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Mon, 23 Dec 2024 23:47:09 +0700 Subject: [PATCH 09/11] fixup! fixup! fixup! fixup! fixup! TW-2165: Improve get/store Participants --- lib/src/room.dart | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/src/room.dart b/lib/src/room.dart index bd4992f02..902f8ff34 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1425,6 +1425,9 @@ class Room { []; if (cache) { + for (final user in users) { + setState(user); + } await client.database?.storeUsers(users, this); } From 878b5e873c5196ae0207db762a3253f66a4c337b Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Mon, 30 Dec 2024 10:43:23 +0700 Subject: [PATCH 10/11] fixup! fixup! fixup! fixup! fixup! fixup! TW-2165: Improve get/store Participants --- test/database_test/store_user_test.dart | 187 ++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 test/database_test/store_user_test.dart diff --git a/test/database_test/store_user_test.dart b/test/database_test/store_user_test.dart new file mode 100644 index 000000000..35c4811b0 --- /dev/null +++ b/test/database_test/store_user_test.dart @@ -0,0 +1,187 @@ +import 'package:matrix/matrix.dart'; +import 'package:test/test.dart'; + +import '../fake_database.dart'; + +void main() { + group('Store user test\n', () { + late DatabaseApi database; + late Room room; + + setUp(() async { + database = await getHiveCollectionsDatabase(null); + room = Room(id: '!testroom:example.com', client: Client('testclient')); + }); + + test( + 'Give a user\n' + 'When store user is called\n' + 'Then the user is stored in the database', + () async { + final users = [ + User( + '@bob:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ) + ]; + + await database.storeUsers(users, room); + + final storedUser = await database.getUsers(room); + + expect(storedUser.length, 1); + + expect( + storedUser.where((user) => user.id == '@bob:example.org').isNotEmpty, + true, + ); + }, + ); + + test( + 'Give list user (Bob_1, Bob_2, Bob_3, Bob_4)\n' + 'When store users is called\n' + 'Then all user is stored in the database', + () async { + final users = [ + User( + '@bob_1:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ), + User( + '@bob_2:example.org', + displayName: 'Bob_2', + avatarUrl: 'mxc://example.com', + room: room, + ), + User( + '@bob_3:example.org', + displayName: 'Bob_3', + avatarUrl: 'mxc://example.com', + room: room, + ), + User( + '@bob_4:example.org', + displayName: 'Bob_4', + avatarUrl: 'mxc://example.com', + room: room, + ), + ]; + + await database.storeUsers(users, room); + + final storedUser = await database.getUsers(room); + + expect(storedUser.length, 5); + + expect( + storedUser.where((user) => user.id == '@bob:example.org').isNotEmpty, + true, + ); + + expect( + storedUser + .where((user) => user.id == '@bob_1:example.org') + .isNotEmpty, + true, + ); + + expect( + storedUser + .where((user) => user.id == '@bob_2:example.org') + .isNotEmpty, + true, + ); + + expect( + storedUser + .where((user) => user.id == '@bob_3:example.org') + .isNotEmpty, + true, + ); + + expect( + storedUser + .where((user) => user.id == '@bob_4:example.org') + .isNotEmpty, + true, + ); + }, + ); + + test( + 'Give list user have 2 users duplicated\n' + 'When store users is called\n' + 'Then only one user is stored in the database', + () async { + final users = [ + User( + '@bob_5:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ), + User( + '@bob_5:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ), + ]; + + await database.storeUsers(users, room); + + final storedUser = await database.getUsers(room); + + expect(storedUser.length, 6); + + expect( + storedUser.where((user) => user.id == '@bob_5:example.org').length, + 1, + ); + }, + ); + + test( + 'Give a user with id is @bob_5:example.org\n' + 'When the user existed in the database\n' + 'AND store users is called\n' + 'Then cannot store the user in the database', + () async { + final getUserInitial = await database.getUsers(room); + + expect(getUserInitial.length, 6); + + expect( + getUserInitial + .where((user) => user.id == '@bob_5:example.org') + .length, + 1, + ); + final users = [ + User( + '@bob_5:example.org', + displayName: 'Bob', + avatarUrl: 'mxc://example.com', + room: room, + ), + ]; + + await database.storeUsers(users, room); + + final storedUser = await database.getUsers(room); + + expect(storedUser.length, 6); + + expect( + storedUser.where((user) => user.id == '@bob_5:example.org').length, + 1, + ); + }, + ); + }); +} From 5db9aac805dc165508ee10abfe06a4489b98f26a Mon Sep 17 00:00:00 2001 From: HuyNguyen Date: Fri, 3 Jan 2025 14:31:32 +0700 Subject: [PATCH 11/11] fixup! fixup! fixup! fixup! fixup! fixup! fixup! TW-2165: Improve get/store Participants --- lib/src/room.dart | 6 +++++- test/database_test/store_user_test.dart | 23 ++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/lib/src/room.dart b/lib/src/room.dart index 902f8ff34..a4c5eefc4 100644 --- a/lib/src/room.dart +++ b/lib/src/room.dart @@ -1428,7 +1428,11 @@ class Room { for (final user in users) { setState(user); } - await client.database?.storeUsers(users, this); + try { + await client.database?.storeUsers(users, this); + } catch (e) { + Logs().w('Room::requestParticipantsFromServer: Unable to store users in the database', e); + } } _requestedParticipants = cache; diff --git a/test/database_test/store_user_test.dart b/test/database_test/store_user_test.dart index 35c4811b0..ef51ff1f9 100644 --- a/test/database_test/store_user_test.dart +++ b/test/database_test/store_user_test.dart @@ -13,6 +13,10 @@ void main() { room = Room(id: '!testroom:example.com', client: Client('testclient')); }); + tearDown(() async { + await database.close(); + }); + test( 'Give a user\n' 'When store user is called\n' @@ -76,12 +80,7 @@ void main() { final storedUser = await database.getUsers(room); - expect(storedUser.length, 5); - - expect( - storedUser.where((user) => user.id == '@bob:example.org').isNotEmpty, - true, - ); + expect(storedUser.length, 4); expect( storedUser @@ -137,7 +136,7 @@ void main() { final storedUser = await database.getUsers(room); - expect(storedUser.length, 6); + expect(storedUser.length, 1); expect( storedUser.where((user) => user.id == '@bob_5:example.org').length, @@ -154,14 +153,8 @@ void main() { () async { final getUserInitial = await database.getUsers(room); - expect(getUserInitial.length, 6); + expect(getUserInitial.length, 0); - expect( - getUserInitial - .where((user) => user.id == '@bob_5:example.org') - .length, - 1, - ); final users = [ User( '@bob_5:example.org', @@ -175,7 +168,7 @@ void main() { final storedUser = await database.getUsers(room); - expect(storedUser.length, 6); + expect(storedUser.length, 1); expect( storedUser.where((user) => user.id == '@bob_5:example.org').length,