Skip to content

Commit

Permalink
perf(messagelist): Upgrade bulk messages to multi-thread with index w…
Browse files Browse the repository at this point in the history 
…orker
  • Loading branch information
@castaway
castaway committed Jan 7, 2025
1 parent ce7a552 commit 798ed9e
Show file tree
Hide file tree
Showing 12 changed files with 120 additions and 87 deletions.
20 changes: 10 additions & 10 deletions e2e/cypress/integration/mailviewer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ describe('Interacting with mailviewer', () => {
// });

it('can open an email and go back and forth in browser history', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/#Inbox:11');

cy.wait('@get11', {'timeout':10000});
Expand All @@ -45,7 +45,7 @@ describe('Interacting with mailviewer', () => {
});

it('can reply to an email with no "To"', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/#Inbox:11')
cy.wait('@get11', {'timeout':10000});
// cy.get('#messageContents');
Expand All @@ -59,7 +59,7 @@ describe('Interacting with mailviewer', () => {
});

it('can forward an email with no "To"', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/');
cy.wait('@get11', {'timeout':10000});
cy.visit('/#Inbox:11');
Expand All @@ -74,7 +74,7 @@ describe('Interacting with mailviewer', () => {
});

it('can reply to an email with no "To" or "Subject"', () => {
cy.intercept('/rest/v1/email/13').as('get13');
cy.intercept('/rest/v1/email/download/*').as('get13');
cy.visit('/');
cy.wait('@get13', {'timeout':10000});
cy.visit('/#Inbox:13');
Expand All @@ -89,7 +89,7 @@ describe('Interacting with mailviewer', () => {
});

it('can forward an email with no "To" or "Subject"', () => {
cy.intercept('/rest/v1/email/13').as('get13');
cy.intercept('/rest/v1/email/download/*').as('get13');
cy.visit('/');
cy.wait('@get13', {'timeout':10000});
cy.visit('/#Inbox:13');
Expand All @@ -104,7 +104,7 @@ describe('Interacting with mailviewer', () => {
});

it('Vertical to horizontal mode exposes full height button', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/');
cy.wait('@get11', {'timeout':10000});
cy.visit('/#Inbox:11');
Expand All @@ -116,7 +116,7 @@ describe('Interacting with mailviewer', () => {
});

it('Changing viewpane height is stored', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/');
cy.wait('@get11', {'timeout':10000});
cy.visit('/#Inbox:11');
Expand All @@ -133,7 +133,7 @@ describe('Interacting with mailviewer', () => {
});

it('Half height reduces stored pane height', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/');
cy.wait('@get11', {'timeout':10000});
cy.visit('/#Inbox:11');
Expand Down Expand Up @@ -161,7 +161,7 @@ describe('Interacting with mailviewer', () => {
});

it('Revisit open email in horizontal mode loads it', () => {
cy.intercept('/rest/v1/email/11').as('get11');
cy.intercept('/rest/v1/email/download/*').as('get11');
cy.visit('/');
cy.wait('@get11', {'timeout':10000});
cy.visit('/#Inbox:11');
Expand All @@ -177,7 +177,7 @@ describe('Interacting with mailviewer', () => {
});

it('Can go out of mailviewer and back and still see our email', () => {
cy.intercept('/rest/v1/email/12').as('get12');
cy.intercept('/rest/v1/email/download/*').as('get12');
cy.visit('/');
cy.wait('@get12',{'timeout':10000});
cy.visit('/#Inbox:12');
Expand Down
4 changes: 2 additions & 2 deletions e2e/cypress/integration/message-caching.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ describe('Message caching', () => {

});

it('should cache all messages on first time page load', () => {
it('should fetch all messages on first time page load', () => {
cy.intercept('/rest/v1/email/download/*').as('message12requested');

cy.visit('/');
Expand All @@ -18,7 +18,7 @@ describe('Message caching', () => {
});

it('should not re-request messages after a page reload', () => {
cy.intercept('/rest/v1/email/12').as('message12requested');
cy.intercept('/rest/v1/email/download/*').as('message12requested');

cy.visit('/');
cy.wait('@message12requested', {'timeout':10000});
Expand Down
9 changes: 6 additions & 3 deletions e2e/mockserver/mockserver.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ END:VCALENDAR
'status': 'success',
'result': messages,
}));
return;
}

const emailendpoint = requesturl.match(/\/rest\/v1\/email\/([0-9]+)/);
Expand Down Expand Up @@ -994,28 +995,32 @@ END:VCALENDAR

getMessage(mailid: number): any {
let message_obj = mail_message_obj;
message_obj.status = 'success';
if (mailid === 11) {
message_obj = JSON.parse(JSON.stringify(mail_message_obj));
const to = message_obj.result.headers['to'];
delete message_obj.result.headers['to'];
message_obj.result.headers['cc'] = to;
message_obj.result.headers['subject'] = "No 'To', just 'CC'";
message_obj.result.mid = mailid;
}
if (mailid === 12) {
message_obj = JSON.parse(JSON.stringify(mail_message_obj));
message_obj.result.headers['to'].value[0].address = "[email protected]";
message_obj.result.headers['to'].text = "[email protected]";
message_obj.result.headers['subject'] = "Default from fix test";
message_obj.result.mid = mailid;
}
if (mailid === 13) {
message_obj = JSON.parse(JSON.stringify(mail_message_obj));
const to = message_obj.result.headers['to'];
delete message_obj.result.headers['to'];
message_obj.result.headers['cc'] = to;
message_obj.result.headers['subject'] = "";
message_obj.result.mid = mailid;
}
// This one warns, we couldnt find it!
if (mailid === '14') {
if (mailid === 14) {
message_obj = {
'status':'warning',
'errors': [
Expand All @@ -1024,8 +1029,6 @@ END:VCALENDAR
};
}

message_obj.status = 'success';

return message_obj;
}
}
14 changes: 11 additions & 3 deletions src/app/app.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
} from './canvastable/canvastable';
import { SingleMailViewerComponent } from './mailviewer/singlemailviewer.component';
import { SearchService } from './xapian/searchservice';
import { PostMessageAction } from './xapian/messageactions';

import { MatLegacyDialogRef as MatDialogRef, MatLegacyDialog as MatDialog } from '@angular/material/legacy-dialog';
import { MatIconRegistry } from '@angular/material/icon';
Expand Down Expand Up @@ -464,12 +465,19 @@ export class AppComponent implements OnInit, AfterViewInit, CanvasTableSelectLis
const messageIds = rowIndexes.filter(
idx => idx < this.canvastable.rows.rowCount()
).map(idx => this.canvastable.rows.getRowMessageId(idx));
// FIXME: promise errors?
this.rmmapi.downloadMessages(messageIds).then(
(messages) => {
for (const msg messages) {
this.searchService.updateMessageText(parseInt(msg['id'], 10));
const updateWorker = new Map();
for (const msg of messages) {
this.searchService.updateMessageText(msg['mid']);
updateWorker.set(msg['mid'], msg.text.text);
};
// Send to the messageCache in the worker, so we can add the text to the index:
if(updateWorker.size > 0) {
this.searchService.indexWorker.postMessage({'action': PostMessageAction.messageCache, 'updates': updateWorker });
this.canvastable.hasChanges = true;
}
this.canvastable.hasChanges = true;
});
});

Expand Down
3 changes: 1 addition & 2 deletions src/app/mailviewer/singlemailviewer.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -461,7 +461,6 @@ export class SingleMailViewerComponent implements OnInit, DoCheck, AfterViewInit
}

private processMessageContents(messageContents: MessageContents): Mail {
console.log('processMessageContents processing', messageContents);
const res: any = Object.assign({}, messageContents);
if (res.status === 'warning') {
// status === 'error' already displayed in showBackendErrors?
Expand Down Expand Up @@ -500,7 +499,7 @@ export class SingleMailViewerComponent implements OnInit, DoCheck, AfterViewInit

res.sanitized_html = this.expandAttachmentData(res.attachments, res.sanitized_html);
res.sanitized_html_without_images = this.expandAttachmentData(res.attachments, res.sanitized_html_without_images);
res.visible_attachment_count = res.attachments.filter((att) => !att.inte
res.visible_attachment_count = res.attachments.filter((att) => !att.inteinternal).length;

// Remove style tag otherwise angular sanitazion will display style tag content as text

Expand Down
12 changes: 12 additions & 0 deletions src/app/rmmapi/lru-message-cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ export class LRUMessageCache<T> {
}
}

checkIds(ids: number[]): number[] {
const keys = [...this.messages.keys()];
const matches = keys.filter((val) => ids.includes(val));
return matches;
}

get(id: number): T {
const row = this.messages.get(id);
if (row) {
Expand All @@ -50,6 +56,12 @@ export class LRUMessageCache<T> {
return row?.message;
}

getMany(ids: number[]): T[] {
const found = this.checkIds(ids);
const results = found.map((id) => this.get(id));
return results;
}

delete(id: number): void {
this.messages.delete(id);
}
Expand Down
6 changes: 3 additions & 3 deletions src/app/rmmapi/messagecache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ export class MessageCache {
this.db.version(3).stores({
// use out-of-line keys, but index "id"
// yes, empty first arg is deliberate
messages: ',&id',
messages: ',&mid',
});
} catch (err) {
console.log(`Error initializing messagecache: ${err}`);
Expand All @@ -50,7 +50,7 @@ export class MessageCache {
// verify which ids we already have
async checkIds(ids: number[]): Promise<number[]> {
return this.db?.table('messages')
.where('id')
.where('mid')
.anyOf(ids)
.primaryKeys()
.then(
Expand All @@ -60,7 +60,7 @@ export class MessageCache {

async getMany(ids: number[]): Promise<MessageContents[]> {
return this.db?.table('messages')
.where('id')
.where('mid')
.anyOf(ids)
.toArray()
.then(
Expand Down
2 changes: 1 addition & 1 deletion src/app/rmmapi/rbwebmail.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { FolderListEntry } from '../common/folderlistentry';
import { MatLegacyDialogModule as MatDialogModule } from '@angular/material/legacy-dialog';
import { MatLegacySnackBarModule as MatSnackBarModule } from '@angular/material/legacy-snack-bar';
import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing';
import { MessageCache } from './messagecache';
import { firstValueFrom } from 'rxjs';

describe('RBWebMail', () => {
Expand Down Expand Up @@ -85,7 +86,6 @@ describe('RBWebMail', () => {

messageContentsObservable = rmmapi.getMessageContents(123, true);
await new Promise(resolve => setTimeout(resolve, 0));
httpTestingController.expectOne('/rest/v1/email/123').flush({
req = httpTestingController.expectOne('/rest/v1/email/123');
req.flush({
status: 'success',
Expand Down
Loading

0 comments on commit 798ed9e

Please sign in to comment.