From 8518cf23a4d75abf9a482aef21664c760e42b05f Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Mon, 25 Jan 2021 19:17:40 -0500 Subject: [PATCH 01/14] initial work on sending history in MUCs --- src/xmppjs/HistoryManager.ts | 105 +++++++++++++++++++++++++++++++++++ src/xmppjs/XJSGateway.ts | 40 +++++++++++-- src/xmppjs/XJSInstance.ts | 1 + 3 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 src/xmppjs/HistoryManager.ts diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts new file mode 100644 index 00000000..2da3457f --- /dev/null +++ b/src/xmppjs/HistoryManager.ts @@ -0,0 +1,105 @@ +import { Element } from "@xmpp/xml"; +import { JID } from "@xmpp/jid"; + +export interface IHistoryLimits { + maxChars?: number | undefined, + maxStanzas?: number | undefined, + seconds?: number | undefined, + since?: Date | undefined, +} + +/** + * Abstraction of a history storage backend. + */ +interface IHistoryStorage { + // add a message to the history of a given room + addMessage: (chatName: string, message: Element, jid: JID) => Promise; + // get the history of a room. The storage should apply the limits that it + // wishes too, and remove the limits that it applied from the `limits` + // parameter. The returned list of Elements must include the Delayed + // Delivery information. + getHistory: (chatName: string, limits: IHistoryLimits) => Promise; +} + +// pad a number to be two digits long +function pad2(n: number) { + return n < 10 ? "0" + n.toString() : n.toString(); +} + +/** + * Store room history in memory. + */ +export class MemoryStorage implements IHistoryStorage { + private history: Map; + constructor(public maxHistory: number) { + this.history = new Map(); + } + + async addMessage(chatName: string, message: Element, jid: JID): Promise { + if (!this.history.has(chatName)) { + this.history.set(chatName, []); + } + const currRoomHistory = this.history.get(chatName); + + // shallow-copy the message, and add the timestamp + const copiedMessage = new Element(message.name, message.attrs); + copiedMessage.append(message.children as Element[]); + copiedMessage.attr("from", jid.toString()); + const now = new Date(); + copiedMessage.append(new Element("delay", { + xmlns: "urn:xmpp:delay", + from: chatName, + // based on the polyfill at + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString + stamp: now.getUTCFullYear() + + "-" + pad2(now.getUTCMonth() + 1) + + "-" + pad2(now.getUTCDate()) + + "T" + pad2(now.getUTCHours()) + + ":" + pad2(now.getUTCMinutes()) + + ":" + pad2(now.getUTCSeconds()) + "Z", + })); + + currRoomHistory.push(copiedMessage); + + while (currRoomHistory.length > this.maxHistory) { + currRoomHistory.shift(); + } + } + + async getHistory(chatName: string, limits: IHistoryLimits) { + return this.history.get(chatName) || []; + } +} + +/** + * Manage room history for a MUC + */ +export class HistoryManager { + constructor( + private storage: IHistoryStorage, + ) {} + + async addMessage(chatName: string, message: Element, jid: JID): Promise { + console.log("adding message for room", chatName, message, jid); + return this.storage.addMessage(chatName, message, jid); + } + + async getHistory(chatName: string, limits: IHistoryLimits): Promise { + console.log("getting messages for room", chatName); + if (limits.seconds) { + const since = new Date(Date.now() - limits.seconds * 1000); + if (limits.since === undefined || limits.since < since) { + limits.since = since; + } + delete limits.seconds; + } + let history: Element[] = await this.storage.getHistory(chatName, limits); + + if ("maxStanzas" in limits && history.length > limits.maxStanzas) { + history = history.slice(-limits.maxStanzas); + } + // FIXME: filter by since, maxchars + console.log(history); + return history; + } +} diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index ad46775c..d149df40 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -1,10 +1,17 @@ import { XmppJsInstance, XMPP_PROTOCOL } from "./XJSInstance"; import { Element, x } from "@xmpp/xml"; +import parse from "@xmpp/xml/lib/parse"; import { jid, JID } from "@xmpp/jid"; import { Logging } from "matrix-appservice-bridge"; import { IConfigBridge } from "../Config"; import { IBasicProtocolMessage } from "..//MessageFormatter"; -import { IGatewayJoin, IUserStateChanged, IStoreRemoteUser, IUserInfo } from "../bifrost/Events"; +import { + IGatewayJoin, + IUserStateChanged, + IStoreRemoteUser, + IUserInfo, + IReceivedImMsg +} from "../bifrost/Events"; import { IGatewayRoom } from "../bifrost/Gateway"; import { PresenceCache } from "./PresenceCache"; import { XHTMLIM } from "./XHTMLIM"; @@ -17,6 +24,7 @@ import { XMPPStatusCode } from "./XMPPConstants"; import { AutoRegistration } from "../AutoRegistration"; import { GatewayStateResolve } from "./GatewayStateResolve"; import { MatrixMembershipEvent } from "../MatrixTypes"; +import { IHistoryLimits, HistoryManager, MemoryStorage } from "./HistoryManager"; const log = Logging.get("XmppJsGateway"); @@ -31,18 +39,24 @@ export interface RemoteGhostExtraData { * and XMPP. */ export class XmppJsGateway implements IGateway { - // For storing room history, should be clipped at MAX_HISTORY per room. - private roomHistory: Map; + // For storing room history + private roomHistory: HistoryManager; // For storing requests to be responded to, like joins private stanzaCache: Map; // id -> stanza private presenceCache: PresenceCache; // Storing every XMPP user and their anonymous. private members: GatewayMUCMembership; constructor(private xmpp: XmppJsInstance, private registration: AutoRegistration, private config: IConfigBridge) { - this.roomHistory = new Map(); + this.roomHistory = new HistoryManager(new MemoryStorage(50)); this.stanzaCache = new Map(); this.members = new GatewayMUCMembership(); this.presenceCache = new PresenceCache(true); + xmpp.on("received-chat-msg-xmpp", (convName: string, stanza: Element) => { + this.roomHistory.addMessage( + convName, stanza, + this.members.getXmppMemberByDevice(convName, stanza.attrs.from).anonymousJid, + ); + }); } public handleStanza(stanza: Element, gatewayAlias: string) { @@ -163,6 +177,16 @@ export class XmppJsGateway implements IGateway { "groupchat", ) ); + + // add the message to the room history + const historyStanza = new StzaMessage( + from.anonymousJid.toString(), + "", + msg, + "groupchat", + ); + this.roomHistory.addMessage(chatName, parse(historyStanza.xml), from.anonymousJid); + return this.xmpp.xmppSendBulk(msgs); } @@ -419,10 +443,14 @@ export class XmppJsGateway implements IGateway { // 4. Room history log.debug("Emitting history"); - const history: Element[] = this.roomHistory.get(room.roomId) || []; + const historyLimits: IHistoryLimits = {}; + const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); + if (historyRequest !== undefined) { + console.log(historyRequest); + } + const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); history.forEach((e) => { e.attrs.to = stanza.attrs.from; - // TODO: Add delay info to this. this.xmpp.xmppWriteToStream(e); }); diff --git a/src/xmppjs/XJSInstance.ts b/src/xmppjs/XJSInstance.ts index bf36c175..770a240b 100644 --- a/src/xmppjs/XJSInstance.ts +++ b/src/xmppjs/XJSInstance.ts @@ -753,6 +753,7 @@ export class XmppJsInstance extends EventEmitter implements IBifrostInstance { if (type === "groupchat") { log.debug("Emitting group message", message); + this.emit("received-chat-msg-xmpp", convName, stanza); this.emit("received-chat-msg", { eventName: "received-chat-msg", sender: from.toString(), From a01c24f5e2b366fc8b5b69b785373480600ad828 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Mon, 25 Jan 2021 19:22:47 -0500 Subject: [PATCH 02/14] remove console.logs --- src/xmppjs/HistoryManager.ts | 3 --- src/xmppjs/XJSGateway.ts | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts index 2da3457f..e94d0e72 100644 --- a/src/xmppjs/HistoryManager.ts +++ b/src/xmppjs/HistoryManager.ts @@ -80,12 +80,10 @@ export class HistoryManager { ) {} async addMessage(chatName: string, message: Element, jid: JID): Promise { - console.log("adding message for room", chatName, message, jid); return this.storage.addMessage(chatName, message, jid); } async getHistory(chatName: string, limits: IHistoryLimits): Promise { - console.log("getting messages for room", chatName); if (limits.seconds) { const since = new Date(Date.now() - limits.seconds * 1000); if (limits.since === undefined || limits.since < since) { @@ -99,7 +97,6 @@ export class HistoryManager { history = history.slice(-limits.maxStanzas); } // FIXME: filter by since, maxchars - console.log(history); return history; } } diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index d149df40..4196de6f 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -446,7 +446,7 @@ export class XmppJsGateway implements IGateway { const historyLimits: IHistoryLimits = {}; const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); if (historyRequest !== undefined) { - console.log(historyRequest); + // FIXME: actually set limits } const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); history.forEach((e) => { From 9fce998369b6c0da592bf5bb5614890cf95fbd6f Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 28 Jan 2021 12:54:38 -0500 Subject: [PATCH 03/14] more improvements, and actually use the client's history management --- src/xmppjs/HistoryManager.ts | 57 ++++++++++++++++++++---------------- src/xmppjs/XJSGateway.ts | 28 +++++++++++++++++- 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts index e94d0e72..4a4b640b 100644 --- a/src/xmppjs/HistoryManager.ts +++ b/src/xmppjs/HistoryManager.ts @@ -2,10 +2,10 @@ import { Element } from "@xmpp/xml"; import { JID } from "@xmpp/jid"; export interface IHistoryLimits { - maxChars?: number | undefined, - maxStanzas?: number | undefined, - seconds?: number | undefined, - since?: Date | undefined, + maxchars?: number, + maxstanzas?: number, + seconds?: number, + since?: Date, } /** @@ -13,7 +13,7 @@ export interface IHistoryLimits { */ interface IHistoryStorage { // add a message to the history of a given room - addMessage: (chatName: string, message: Element, jid: JID) => Promise; + addMessage: (chatName: string, message: Element, jid: JID) => unknown; // get the history of a room. The storage should apply the limits that it // wishes too, and remove the limits that it applied from the `limits` // parameter. The returned list of Elements must include the Delayed @@ -21,11 +21,6 @@ interface IHistoryStorage { getHistory: (chatName: string, limits: IHistoryLimits) => Promise; } -// pad a number to be two digits long -function pad2(n: number) { - return n < 10 ? "0" + n.toString() : n.toString(); -} - /** * Store room history in memory. */ @@ -35,7 +30,7 @@ export class MemoryStorage implements IHistoryStorage { this.history = new Map(); } - async addMessage(chatName: string, message: Element, jid: JID): Promise { + addMessage(chatName: string, message: Element, jid: JID): void { if (!this.history.has(chatName)) { this.history.set(chatName, []); } @@ -45,18 +40,10 @@ export class MemoryStorage implements IHistoryStorage { const copiedMessage = new Element(message.name, message.attrs); copiedMessage.append(message.children as Element[]); copiedMessage.attr("from", jid.toString()); - const now = new Date(); copiedMessage.append(new Element("delay", { xmlns: "urn:xmpp:delay", from: chatName, - // based on the polyfill at - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString - stamp: now.getUTCFullYear() + - "-" + pad2(now.getUTCMonth() + 1) + - "-" + pad2(now.getUTCDate()) + - "T" + pad2(now.getUTCHours()) + - ":" + pad2(now.getUTCMinutes()) + - ":" + pad2(now.getUTCSeconds()) + "Z", + stamp: (new Date()).toISOString(), })); currRoomHistory.push(copiedMessage); @@ -66,11 +53,14 @@ export class MemoryStorage implements IHistoryStorage { } } - async getHistory(chatName: string, limits: IHistoryLimits) { + async getHistory(chatName: string, limits: IHistoryLimits): Promise { return this.history.get(chatName) || []; } } +// TODO: make a class that stores in PostgreSQL so that we don't lose history +// when we restart + /** * Manage room history for a MUC */ @@ -79,7 +69,7 @@ export class HistoryManager { private storage: IHistoryStorage, ) {} - async addMessage(chatName: string, message: Element, jid: JID): Promise { + addMessage(chatName: string, message: Element, jid: JID): unknown { return this.storage.addMessage(chatName, message, jid); } @@ -93,10 +83,27 @@ export class HistoryManager { } let history: Element[] = await this.storage.getHistory(chatName, limits); - if ("maxStanzas" in limits && history.length > limits.maxStanzas) { - history = history.slice(-limits.maxStanzas); + if ("maxstanzas" in limits && history.length > limits.maxstanzas) { + history = history.slice(-limits.maxstanzas); } - // FIXME: filter by since, maxchars + + if ("since" in limits) { + // FIXME: binary search would be better than linear search + const idx = history.findIndex((el) => { + try { + const ts = el.getChild("delay", "urn:xmpp:delay")?.attr("stamp"); + console.log(ts); + return new Date(Date.parse(ts)) >= limits.since; + } catch (e) { + return false; + } + }); + if (idx > 0) { + history = history.slice(idx); + } + } + // FIXME: filter by maxchars + return history; } } diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index 4196de6f..2dc647d3 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -446,7 +446,33 @@ export class XmppJsGateway implements IGateway { const historyLimits: IHistoryLimits = {}; const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); if (historyRequest !== undefined) { - // FIXME: actually set limits + const getIntValue = (x) => { + if (!x.match(/^\d+$/)) { + throw new Error("Not a number"); + } + return parseInt(x); + }; + const getDateValue = (x) => { + const val = Date.parse(x); + if (isNaN(val)) { + throw new Error("Not a date"); + } + return new Date(val); // Date.parse returns a number, which we need to turn into a Date object + }; + const getHistoryParam = (name: string, parser: (string) => any): void => { + const param = historyRequest.getAttr(name); + if (param !== undefined) { + try { + historyLimits[name] = parser(param); + } catch (e) { + log.debug(`Invalid ${name} in history management: "${param}" (${e})`); + } + } + }; + getHistoryParam("maxchars", getIntValue); + getHistoryParam("maxstanzas", getIntValue); + getHistoryParam("seconds", getIntValue); + getHistoryParam("since", getDateValue); } const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); history.forEach((e) => { From 58e8bd101e2e7ac3ff1953456a10dfc89b008fda Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 28 Jan 2021 17:32:30 -0500 Subject: [PATCH 04/14] apply all history limits --- src/xmppjs/HistoryManager.ts | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts index 4a4b640b..4b597465 100644 --- a/src/xmppjs/HistoryManager.ts +++ b/src/xmppjs/HistoryManager.ts @@ -83,26 +83,37 @@ export class HistoryManager { } let history: Element[] = await this.storage.getHistory(chatName, limits); + // index of the first history element that we will keep after applying + // the limits + let idx = 0; + if ("maxstanzas" in limits && history.length > limits.maxstanzas) { - history = history.slice(-limits.maxstanzas); + idx = history.length - limits.maxstanzas; } if ("since" in limits) { // FIXME: binary search would be better than linear search - const idx = history.findIndex((el) => { + for (; idx < history.length; idx++) { try { - const ts = el.getChild("delay", "urn:xmpp:delay")?.attr("stamp"); - console.log(ts); - return new Date(Date.parse(ts)) >= limits.since; - } catch (e) { - return false; - } - }); - if (idx > 0) { - history = history.slice(idx); + const ts = history[idx].getChild("delay", "urn:xmpp:delay")?.attr("stamp"); + if (new Date(Date.parse(ts)) >= limits.since) { + break; + } + } catch (e) {} + } + } + + if ("maxchars" in limits) { + let numChars = 0; + let i = history.length; + for (; i > idx && numChars < limits.maxchars; i--) { + numChars += history[i - 1].toString().length; } } - // FIXME: filter by maxchars + + if (idx > 0) { + history = history.slice(idx); + } return history; } From 850e3135abcf18adaf506d4bf487a2f164a43729 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 28 Jan 2021 20:11:36 -0500 Subject: [PATCH 05/14] fix lint --- src/xmppjs/XJSGateway.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index 2dc647d3..b05b5587 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -446,20 +446,20 @@ export class XmppJsGateway implements IGateway { const historyLimits: IHistoryLimits = {}; const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); if (historyRequest !== undefined) { - const getIntValue = (x) => { - if (!x.match(/^\d+$/)) { + const getIntValue = (str) => { + if (!str.match(/^\d+$/)) { throw new Error("Not a number"); } - return parseInt(x); + return parseInt(str); }; - const getDateValue = (x) => { - const val = Date.parse(x); + const getDateValue = (str) => { + const val = Date.parse(str); if (isNaN(val)) { throw new Error("Not a date"); } return new Date(val); // Date.parse returns a number, which we need to turn into a Date object }; - const getHistoryParam = (name: string, parser: (string) => any): void => { + const getHistoryParam = (name: string, parser: (str: string) => any): void => { const param = historyRequest.getAttr(name); if (param !== undefined) { try { From eeb8cad5739e1ea5ecae75c62abf8cb9881f86b7 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 28 Jan 2021 22:22:39 -0500 Subject: [PATCH 06/14] add changelog --- changelog.d/227.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/227.feature diff --git a/changelog.d/227.feature b/changelog.d/227.feature new file mode 100644 index 00000000..4e1aed57 --- /dev/null +++ b/changelog.d/227.feature @@ -0,0 +1 @@ +Send room history to XMPP clients when they join. From ee9036398157371022fc337007821c16737d1198 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Fri, 29 Jan 2021 12:45:09 -0500 Subject: [PATCH 07/14] set a default history limit if none provided by the client --- src/xmppjs/XJSGateway.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index b05b5587..9bbdf15a 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -473,6 +473,9 @@ export class XmppJsGateway implements IGateway { getHistoryParam("maxstanzas", getIntValue); getHistoryParam("seconds", getIntValue); getHistoryParam("since", getDateValue); + } else { + // default to 20 stanzas if the client doesn't specify + historyLimits.maxstanzas = 20; } const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); history.forEach((e) => { From cefb56e3801b3b8097b60c933936a1ca2b828f53 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 4 Feb 2021 14:40:18 -0500 Subject: [PATCH 08/14] Apply suggestions from code review Co-authored-by: Christian Paul --- src/xmppjs/HistoryManager.ts | 4 ++-- src/xmppjs/XJSGateway.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts index 4b597465..3793df2e 100644 --- a/src/xmppjs/HistoryManager.ts +++ b/src/xmppjs/HistoryManager.ts @@ -96,10 +96,10 @@ export class HistoryManager { for (; idx < history.length; idx++) { try { const ts = history[idx].getChild("delay", "urn:xmpp:delay")?.attr("stamp"); - if (new Date(Date.parse(ts)) >= limits.since) { + if (new Date(ts) >= limits.since) { break; } - } catch (e) {} + } catch {} } } diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index 9bbdf15a..21985c3f 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -447,17 +447,17 @@ export class XmppJsGateway implements IGateway { const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); if (historyRequest !== undefined) { const getIntValue = (str) => { - if (!str.match(/^\d+$/)) { + if (!/^\d+$/.test(str)) { throw new Error("Not a number"); } return parseInt(str); }; const getDateValue = (str) => { - const val = Date.parse(str); + const val = new Date(str); if (isNaN(val)) { throw new Error("Not a date"); } - return new Date(val); // Date.parse returns a number, which we need to turn into a Date object + return val; // Date.parse returns a number, which we need to turn into a Date object }; const getHistoryParam = (name: string, parser: (str: string) => any): void => { const param = historyRequest.getAttr(name); From 75d0ba3af16e6cc9bd8cb379f7b5024925d6fbf1 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 4 Feb 2021 14:43:46 -0500 Subject: [PATCH 09/14] more fixes --- src/xmppjs/HistoryManager.ts | 1 + src/xmppjs/XJSGateway.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/xmppjs/HistoryManager.ts b/src/xmppjs/HistoryManager.ts index 3793df2e..4641c85c 100644 --- a/src/xmppjs/HistoryManager.ts +++ b/src/xmppjs/HistoryManager.ts @@ -109,6 +109,7 @@ export class HistoryManager { for (; i > idx && numChars < limits.maxchars; i--) { numChars += history[i - 1].toString().length; } + idx = i; } if (idx > 0) { diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index 21985c3f..fc2ece7b 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -457,7 +457,7 @@ export class XmppJsGateway implements IGateway { if (isNaN(val)) { throw new Error("Not a date"); } - return val; // Date.parse returns a number, which we need to turn into a Date object + return val; }; const getHistoryParam = (name: string, parser: (str: string) => any): void => { const param = historyRequest.getAttr(name); From 4f1223e763ecde7296180693f3679e6796c8acdf Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 4 Feb 2021 15:51:45 -0500 Subject: [PATCH 10/14] add unit test --- test/xmppjs/test_HistoryManager.ts | 49 ++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/xmppjs/test_HistoryManager.ts diff --git a/test/xmppjs/test_HistoryManager.ts b/test/xmppjs/test_HistoryManager.ts new file mode 100644 index 00000000..7f95ac71 --- /dev/null +++ b/test/xmppjs/test_HistoryManager.ts @@ -0,0 +1,49 @@ +import { expect } from "chai"; +import { MemoryStorage, HistoryManager } from "../../src/xmppjs/HistoryManager"; +import { Element } from "@xmpp/xml"; +import { JID } from "@xmpp/jid"; + +describe("HistoryManager", () => { + describe("MemoryStorage", () => { + it("should return filtered history", async () => { + const historyManager = new HistoryManager(new MemoryStorage(20)); + historyManager.addMessage( + "room1@example.org", new Element("stanza1"), + new JID("room1", "example.org", "user1"), + ); + historyManager.addMessage( + "room1@example.org", new Element("stanza2"), + new JID("room1", "example.org", "user1"), + ); + historyManager.addMessage( + "room1@example.org", new Element("stanza3"), + new JID("room1", "example.org", "user1"), + ); + historyManager.addMessage( + "room2@example.org", new Element("stanza1"), + new JID("room1", "example.org", "user1"), + ); + + const unfilteredRoom1 = await historyManager.getHistory("room1@example.org", {}); + expect(unfilteredRoom1.length).to.equal(3); + const unfilteredRoom2 = await historyManager.getHistory("room2@example.org", {}); + expect(unfilteredRoom2.length).to.equal(1); + + const maxStanzasRoom1 = await historyManager.getHistory("room1@example.org", { + maxstanzas: 2, + }); + expect(maxStanzasRoom1.length).to.equal(2); + const maxStanzasRoom2 = await historyManager.getHistory("room2@example.org", { + maxstanzas: 2, + }); + expect(maxStanzasRoom2.length).to.equal(1); + + // each stanza will be about 40 characters, so maxchars 50 should + // only give us one stanza + const maxCharsRoom1 = await historyManager.getHistory("room1@example.org", { + maxchars: 50, + }); + expect(maxCharsRoom1.length).to.equal(1); + }); + }); +}); From 57cef0bd9c671c8c2003f83d6af367962d0d9d85 Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Thu, 4 Feb 2021 16:03:59 -0500 Subject: [PATCH 11/14] fix build error --- src/xmppjs/XJSGateway.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index fc2ece7b..dbe165e0 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -454,7 +454,9 @@ export class XmppJsGateway implements IGateway { }; const getDateValue = (str) => { const val = new Date(str); - if (isNaN(val)) { + // TypeScript doesn't like giving a Date to isNaN, even though it + // works. And it doesn't like converting directly to number. + if (isNaN(val as unknown as number)) { throw new Error("Not a date"); } return val; From 1eff791d4182399117f12bb9339a510e639701eb Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 8 Mar 2021 12:44:44 +0000 Subject: [PATCH 12/14] Only emit history for gateways --- src/xmppjs/XJSInstance.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xmppjs/XJSInstance.ts b/src/xmppjs/XJSInstance.ts index 770a240b..376f712a 100644 --- a/src/xmppjs/XJSInstance.ts +++ b/src/xmppjs/XJSInstance.ts @@ -561,6 +561,7 @@ export class XmppJsInstance extends EventEmitter implements IBifrostInstance { log.warn(`Message could not be sent, not forwarding to Matrix`); return; } + this.emit("received-chat-msg-xmpp", convName, stanza); // We deliberately do not anonymize the JID here. // We do however strip the resource from = jid(`${from.local}@${from.domain}`); @@ -753,7 +754,6 @@ export class XmppJsInstance extends EventEmitter implements IBifrostInstance { if (type === "groupchat") { log.debug("Emitting group message", message); - this.emit("received-chat-msg-xmpp", convName, stanza); this.emit("received-chat-msg", { eventName: "received-chat-msg", sender: from.toString(), From d82f216838b1a2828c0a6f8fa628718771d0b881 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 8 Mar 2021 12:45:41 +0000 Subject: [PATCH 13/14] Fix history bug --- src/xmppjs/XJSGateway.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index dbe165e0..46786f07 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -52,10 +52,14 @@ export class XmppJsGateway implements IGateway { this.members = new GatewayMUCMembership(); this.presenceCache = new PresenceCache(true); xmpp.on("received-chat-msg-xmpp", (convName: string, stanza: Element) => { - this.roomHistory.addMessage( - convName, stanza, - this.members.getXmppMemberByDevice(convName, stanza.attrs.from).anonymousJid, - ); + try { + this.roomHistory.addMessage( + convName, stanza, + this.members.getXmppMemberByDevice(convName, stanza.attrs.from).anonymousJid, + ); + } catch (ex) { + log.warn(`Failed to add message for ${convName} to history cache`); + } }); } From 703a72df5253272f4f4cf23c128199e1d597585d Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Mon, 8 Mar 2021 12:53:50 +0000 Subject: [PATCH 14/14] Add history visibility settings --- src/GatewayHandler.ts | 7 ++- src/bifrost/Gateway.ts | 1 + src/xmppjs/XJSGateway.ts | 106 +++++++++++++++++---------------- src/xmppjs/XJSInstance.ts | 1 - test/xmppjs/test_XJSGateway.ts | 5 ++ 5 files changed, 68 insertions(+), 52 deletions(-) diff --git a/src/GatewayHandler.ts b/src/GatewayHandler.ts index e26b307f..4ee95c38 100644 --- a/src/GatewayHandler.ts +++ b/src/GatewayHandler.ts @@ -12,6 +12,8 @@ import { BifrostRemoteUser } from "./store/BifrostRemoteUser"; const log = Logging.get("GatewayHandler"); +const HISTORY_SAFE_ENUMS = ['shared', 'world_readable']; + /** * Responsible for handling querys & events on behalf of a gateway style bridge. * The gateway system in the bridge is complex, so pull up a a pew and let's dig in. @@ -58,10 +60,11 @@ export class GatewayHandler { } const promise = (async () => { log.debug(`Getting state for ${roomId}`); - const state = await intent.roomState(roomId, false); + const state = await intent.roomState(roomId); log.debug(`Got state for ${roomId}`); const nameEv = state.find((e) => e.type === "m.room.name"); const topicEv = state.find((e) => e.type === "m.room.topic"); + const historyVis = state.find((e) => e.type === "m.room.history_visibility"); const bot = this.bridge.getBot(); const membership = state.filter((e) => e.type === "m.room.member").map((e: WeakEvent) => ( { @@ -73,6 +76,8 @@ export class GatewayHandler { } )) const room: IGatewayRoom = { + // Default to private + allowHistory: HISTORY_SAFE_ENUMS.includes(historyVis?.content?.history_visibility || 'joined'), name: nameEv ? nameEv.content.name : "", topic: topicEv ? topicEv.content.topic : "", roomId, diff --git a/src/bifrost/Gateway.ts b/src/bifrost/Gateway.ts index 4dbe39cf..1088d96d 100644 --- a/src/bifrost/Gateway.ts +++ b/src/bifrost/Gateway.ts @@ -26,6 +26,7 @@ export interface IGatewayRoom { topic: string; avatar?: string; roomId: string; + allowHistory: boolean; membership: { sender: string; stateKey: string; diff --git a/src/xmppjs/XJSGateway.ts b/src/xmppjs/XJSGateway.ts index 46786f07..687cb174 100644 --- a/src/xmppjs/XJSGateway.ts +++ b/src/xmppjs/XJSGateway.ts @@ -51,16 +51,6 @@ export class XmppJsGateway implements IGateway { this.stanzaCache = new Map(); this.members = new GatewayMUCMembership(); this.presenceCache = new PresenceCache(true); - xmpp.on("received-chat-msg-xmpp", (convName: string, stanza: Element) => { - try { - this.roomHistory.addMessage( - convName, stanza, - this.members.getXmppMemberByDevice(convName, stanza.attrs.from).anonymousJid, - ); - } catch (ex) { - log.warn(`Failed to add message for ${convName} to history cache`); - } - }); } public handleStanza(stanza: Element, gatewayAlias: string) { @@ -189,7 +179,9 @@ export class XmppJsGateway implements IGateway { msg, "groupchat", ); - this.roomHistory.addMessage(chatName, parse(historyStanza.xml), from.anonymousJid); + if (room.allowHistory) { + this.roomHistory.addMessage(chatName, parse(historyStanza.xml), from.anonymousJid); + } return this.xmpp.xmppSendBulk(msgs); } @@ -229,6 +221,16 @@ export class XmppJsGateway implements IGateway { return false; } stanza.attrs.from = preserveFrom; + try { + // TODO: Currently we have no way to determine if this room has private history, + // so we may be adding more strain to the cache than nessacery. + this.roomHistory.addMessage( + chatName, stanza, + member.anonymousJid, + ); + } catch (ex) { + log.warn(`Failed to add message for ${chatName} to history cache`); + } return true; } @@ -446,48 +448,52 @@ export class XmppJsGateway implements IGateway { ); // 4. Room history - log.debug("Emitting history"); - const historyLimits: IHistoryLimits = {}; - const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); - if (historyRequest !== undefined) { - const getIntValue = (str) => { - if (!/^\d+$/.test(str)) { - throw new Error("Not a number"); - } - return parseInt(str); - }; - const getDateValue = (str) => { - const val = new Date(str); - // TypeScript doesn't like giving a Date to isNaN, even though it - // works. And it doesn't like converting directly to number. - if (isNaN(val as unknown as number)) { - throw new Error("Not a date"); - } - return val; - }; - const getHistoryParam = (name: string, parser: (str: string) => any): void => { - const param = historyRequest.getAttr(name); - if (param !== undefined) { - try { - historyLimits[name] = parser(param); - } catch (e) { - log.debug(`Invalid ${name} in history management: "${param}" (${e})`); + if (room.allowHistory) { + log.debug("Emitting history"); + const historyLimits: IHistoryLimits = {}; + const historyRequest = stanza.getChild("x", "http://jabber.org/protocol/muc")?.getChild("history"); + if (historyRequest !== undefined) { + const getIntValue = (str) => { + if (!/^\d+$/.test(str)) { + throw new Error("Not a number"); } - } - }; - getHistoryParam("maxchars", getIntValue); - getHistoryParam("maxstanzas", getIntValue); - getHistoryParam("seconds", getIntValue); - getHistoryParam("since", getDateValue); + return parseInt(str); + }; + const getDateValue = (str) => { + const val = new Date(str); + // TypeScript doesn't like giving a Date to isNaN, even though it + // works. And it doesn't like converting directly to number. + if (isNaN(val as unknown as number)) { + throw new Error("Not a date"); + } + return val; + }; + const getHistoryParam = (name: string, parser: (str: string) => any): void => { + const param = historyRequest.getAttr(name); + if (param !== undefined) { + try { + historyLimits[name] = parser(param); + } catch (e) { + log.debug(`Invalid ${name} in history management: "${param}" (${e})`); + } + } + }; + getHistoryParam("maxchars", getIntValue); + getHistoryParam("maxstanzas", getIntValue); + getHistoryParam("seconds", getIntValue); + getHistoryParam("since", getDateValue); + } else { + // default to 20 stanzas if the client doesn't specify + historyLimits.maxstanzas = 20; + } + const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); + history.forEach((e) => { + e.attrs.to = stanza.attrs.from; + this.xmpp.xmppWriteToStream(e); + }); } else { - // default to 20 stanzas if the client doesn't specify - historyLimits.maxstanzas = 20; + log.debug("Not emitting history, room does not have visibility turned on"); } - const history: Element[] = await this.roomHistory.getHistory(chatName, historyLimits); - history.forEach((e) => { - e.attrs.to = stanza.attrs.from; - this.xmpp.xmppWriteToStream(e); - }); log.debug("Emitting subject"); // 5. The room subject diff --git a/src/xmppjs/XJSInstance.ts b/src/xmppjs/XJSInstance.ts index 376f712a..bf36c175 100644 --- a/src/xmppjs/XJSInstance.ts +++ b/src/xmppjs/XJSInstance.ts @@ -561,7 +561,6 @@ export class XmppJsInstance extends EventEmitter implements IBifrostInstance { log.warn(`Message could not be sent, not forwarding to Matrix`); return; } - this.emit("received-chat-msg-xmpp", convName, stanza); // We deliberately do not anonymize the JID here. // We do however strip the resource from = jid(`${from.local}@${from.domain}`); diff --git a/test/xmppjs/test_XJSGateway.ts b/test/xmppjs/test_XJSGateway.ts index 32d9b0da..4bba9e61 100644 --- a/test/xmppjs/test_XJSGateway.ts +++ b/test/xmppjs/test_XJSGateway.ts @@ -73,6 +73,7 @@ describe("XJSGateway", () => { topic: "GatewayTopic", roomId: "!foo:bar", membership: [], + allowHistory: true, }; try { await gw.onRemoteJoin(null, "myjoinid", room, "@_xmpp_foo:bar"); @@ -93,6 +94,7 @@ describe("XJSGateway", () => { createMember("@_xmpp_baz:bar", "Baz"), createMember("@leavy:bar", "Leavy", "leave"), ], + allowHistory: true, }; gw.handleStanza( x("presence", { @@ -150,6 +152,7 @@ describe("XJSGateway", () => { createMember("@_xmpp_baz:bar", "Baz"), createMember("@leavy:bar", "Leavy", "leave"), ], + allowHistory: true, }; gw.handleStanza( x("presence", { @@ -178,6 +181,7 @@ describe("XJSGateway", () => { topic: "GatewayTopic", roomId: "!foo:bar", membership, + allowHistory: true, }; gw.handleStanza( x("presence", { @@ -202,6 +206,7 @@ describe("XJSGateway", () => { topic: "GatewayTopic", roomId: "!foo:bar", membership: [], + allowHistory: true, }; gw.handleStanza( x("presence", {