Automatically create mapped channels if mappings[...].createRoom is set

changelog.d/1099.bugfix

@@ -0,0 +1 @@
+Add the ability to create rooms automatically for mapped channels in the config.

config.schema.yml

@@ -311,9 +311,13 @@ properties:
                                                     type: "string"
                                                 minItems: 1
                                                 uniqueItems: true
+                                            createRoom:
+                                                type: "boolean"
                                             key:
                                                 type: "string"
-                                        required: ["roomIds"]
+                                        oneOf: 
+                                            - required: ["roomIds"]
+                                            - required: ["createRoom"]
                                         # Legacy format
                                       - type: "array"
                                         items:

spec/integ/static-channels.spec.js

@@ -0,0 +1,84 @@
+const envBundle = require("../util/env-bundle");
+
+describe("Static channels", function() {
+    const staticTestChannel = "#astaticchannel";
+    const generatedTestChannel = "#ageneratedchannel";
+    const generatedRoomId = "!gen:bar";
+    const {env, config, roomMapping, botUserId, test} = envBundle();
+
+    beforeEach(async function() {
+        config.ircService.servers[roomMapping.server].mappings[staticTestChannel] = {
+            roomIds: ["!foo:bar"],
+        };
+
+        config.ircService.servers[roomMapping.server].mappings[generatedTestChannel] = {
+            createRoom: true,
+        };
+
+        await test.beforeEach(env);
+
+        env.ircMock._autoConnectNetworks(
+            roomMapping.server, roomMapping.botNick, roomMapping.server
+        );
+
+        // Ensure rooms are created on startup
+        sdk = env.clientMock._client(botUserId);
+        sdk.createRoom.and.callFake(async function(opts) {
+            return {
+                room_id: generatedRoomId
+            };
+        });
+
+        await test.initEnv(env, config);
+    });
+
+    afterEach(async function() {
+        await test.afterEach(env);
+    });
+
+    it("should insert static channel mappings to bridge store", async function () {
+        const store = await env.ircBridge.getStore();
+        const server = await env.ircBridge.getServer(roomMapping.server);
+        const mappings = await store.getMappingsForChannelByOrigin(server, staticTestChannel, "config");
+        expect(mappings.length).toEqual(1);
+        const entry = mappings[0];
+        expect(entry.matrix.roomId).toEqual("!foo:bar");
+        expect(entry.remote.data).toEqual({
+            domain: roomMapping.server,
+            channel: staticTestChannel,
+            type: "channel",
+        });
+        expect(entry.data.origin).toEqual("config");
+    });
+
+    it("should clear static channel mappings from bridge store", async function () {
+        const store = await env.ircBridge.getStore();
+        const server = await env.ircBridge.getServer(roomMapping.server);
+        await store.removeConfigMappings(server);
+        const mappings = await store.getMappingsForChannelByOrigin(server, staticTestChannel, "config");
+        expect(mappings.length).toEqual(0);
+    });
+
+    it("should create a channel mapping for mappings with createRoom", async function () {
+        const store = await env.ircBridge.getStore();
+        const server = await env.ircBridge.getServer(roomMapping.server);
+        const mappings = await store.getMappingsForChannelByOrigin(server, generatedTestChannel, "config");
+        expect(mappings.length).toEqual(1);
+        const entry = mappings[0];
+        expect(entry.remote.data).toEqual({
+            domain: roomMapping.server,
+            channel: generatedTestChannel,
+            type: "channel",
+        });
+        expect(entry.matrix.roomId).toEqual(generatedRoomId);
+        expect(entry.data.origin).toEqual("config");
+    });
+
+    it("should NOT clear channel mappings for mappings with createRoom", async function () {
+        const store = await env.ircBridge.getStore();
+        const server = await env.ircBridge.getServer(roomMapping.server);
+        await store.removeConfigMappings(server);
+        const mappings = await store.getMappingsForChannelByOrigin(server, generatedTestChannel, "config");
+        expect(mappings.length).toEqual(1);
+    });
+});

src/bridge/IrcBridge.ts

@@ -38,6 +38,7 @@ import { BridgeStateSyncer } from "./BridgeStateSyncer";
 import { Registry } from "prom-client";
 import { spawnMetricsWorker } from "../workers/MetricsWorker";
 import { getBridgeVersion } from "../util/PackageInfo";
+import { trackChannelAndCreateRoom } from "./RoomCreation";
 
 const log = getLogger("IrcBridge");
 const DEFAULT_PORT = 8090;
@@ -368,8 +369,6 @@ export class IrcBridge {
             throw Error("Incorrect database config");
         }
 
-        await this.dataStore.removeConfigMappings();
-
         this.clientPool = new ClientPool(this, this.dataStore);
 
         if (this.config.ircService.debugApi.enabled) {
@@ -502,9 +501,15 @@ export class IrcBridge {
         const requestTimeoutSeconds = this.config.ircService.provisioning.requestTimeoutSeconds;
         this.provisioner = new Provisioner(this, provisioningEnabled, requestTimeoutSeconds);
 
+
+
         log.info("Connecting to IRC networks...");
         await this.connectToIrcNetworks();
 
+        log.info("Creating rooms for mappings with no roomIds");
+        await this.autocreateMappedChannels();
+
+
         promiseutil.allSettled(this.ircServers.map((server) => {
             // Call MODE on all known channels to get modes of all channels
             return Bluebird.cast(this.publicitySyncer.initModes(server));
@@ -613,6 +618,35 @@ export class IrcBridge {
         await promiseutil.allSettled(promises);
     }
 
+    private async autocreateMappedChannels() {
+        const req = new BridgeRequest(new Request());
+        for (const server of this.ircServers) {
+            for (const ircChannel of server.getAutoCreateMappings()) {
+                const existingRooms = await this.dataStore.getMatrixRoomsForChannel(server, ircChannel.channel);
+                if (existingRooms.length > 0) {
+                    // We don't autocreate for existing channels
+                    continue;
+                }
+                // Create a fresh room!
+                try {
+                    const roomAliasName = server.getAliasFromChannel(ircChannel.channel).split(":")[0].substring(1);
+                    await trackChannelAndCreateRoom(this, req, {
+                        server,
+                        ircChannel: ircChannel.channel,
+                        key: ircChannel.key,
+                        origin: "config",
+                        roomAliasName: roomAliasName,
+                        roomVisibility: "public",
+                        shouldTrack: false, // wait for a user to join
+                    });
+                }
+                catch (ex) {
+                    log.warn("Failed to create and track room from config:", ex);
+                }
+            }
+        }
+    }
+
     public async sendMatrixAction(room: MatrixRoom, from: MatrixUser, action: MatrixAction): Promise<void> {
         const intent = this.bridge.getIntent(from.userId);
         if (action.msgType) {

src/bridge/RoomCreation.ts

@@ -3,6 +3,7 @@ import { IrcBridge } from "./IrcBridge";
 import { MatrixRoom, Intent } from "matrix-appservice-bridge";
 import { BridgeRequest } from "../models/BridgeRequest";
 import { RoomOrigin } from "../datastore/DataStore";
+import { IrcRoom } from "../models/IrcRoom";
 
 interface TrackChannelOpts {
     server: IrcServer;
@@ -12,6 +13,9 @@ interface TrackChannelOpts {
     origin: RoomOrigin;
     roomAliasName?: string;
     intent?: Intent;
+    roomVisibility?: "public"|"private";
+    historyVisiblity?: "joined"|"invited"|"shared"|"world_readable";
+    shouldTrack?: boolean;
 }
 
 /**
@@ -21,7 +25,11 @@ interface TrackChannelOpts {
  * @param opts Information about the room creation request.
  */
 export async function trackChannelAndCreateRoom(ircBridge: IrcBridge, req: BridgeRequest, opts: TrackChannelOpts) {
-    const { server, ircChannel, key, inviteList, origin, roomAliasName } = opts;
+    if (opts.shouldTrack === undefined) {
+        opts.shouldTrack = true;
+    }
+    let ircRoom: IrcRoom;
+    const { server, ircChannel, key, inviteList, origin, roomAliasName, shouldTrack } = opts;
     const intent = opts.intent || ircBridge.getAppServiceBridge().getIntent();
     const initialState: ({type: string; state_key: string; content: object})[] = [
         {
@@ -35,7 +43,7 @@ export async function trackChannelAndCreateRoom(ircBridge: IrcBridge, req: Bridg
             type: "m.room.history_visibility",
             state_key: "",
             content: {
-                history_visibility: "joined"
+                history_visibility: opts.historyVisiblity || "joined"
             }
         }
     ];
@@ -56,15 +64,20 @@ export async function trackChannelAndCreateRoom(ircBridge: IrcBridge, req: Bridg
             )
         )
     }
-    req.log.info("Going to track IRC channel %s", ircChannel);
-    const ircRoom = await ircBridge.trackChannel(server, ircChannel, key);
-    req.log.info("Bot is now tracking IRC channel.");
+    if (shouldTrack) {
+        req.log.info("Going to track IRC channel %s", ircChannel);
+        ircRoom = await ircBridge.trackChannel(server, ircChannel, key);
+        req.log.info("Bot is now tracking IRC channel.");
+    }
+    else {
+        ircRoom = new IrcRoom(server, ircChannel);
+    }
     let roomId;
     try {
         const response = await intent.createRoom({
             options: {
                 name: ircChannel,
-                visibility: "private",
+                visibility: opts.roomVisibility || "private",
                 preset: "public_chat",
                 creation_content: {
                     "m.federate": server.shouldFederate()
@@ -85,12 +98,14 @@ export async function trackChannelAndCreateRoom(ircBridge: IrcBridge, req: Bridg
 
     const mxRoom = new MatrixRoom(roomId);
     await ircBridge.getStore().storeRoom(ircRoom, mxRoom, origin);
-    // /mode the channel AFTER we have created the mapping so we process
-    // +s and +i correctly. This is done asyncronously.
-    ircBridge.publicitySyncer.initModeForChannel(server, ircChannel).catch(() => {
-        req.log.error(
-            `Could not init mode for channel ${ircChannel} on ${server.domain}`
-        );
-    });
+    if (shouldTrack) {
+        // /mode the channel AFTER we have created the mapping so we process
+        // +s and +i correctly. This is done asyncronously.
+        ircBridge.publicitySyncer.initModeForChannel(server, ircChannel).catch(() => {
+            req.log.error(
+                `Could not init mode for channel ${ircChannel} on ${server.domain}`
+            );
+        });
+    }
     return { ircRoom, mxRoom };
 }

src/datastore/DataStore.ts

@@ -128,7 +128,7 @@ export interface DataStore {
 
     getRoomIdsFromConfig(): Promise<string[]>;
 
-    removeConfigMappings(): Promise<void>;
+    removeConfigMappings(server: IrcServer): Promise<void>;
 
     getIpv6Counter(): Promise<number>;
 

src/datastore/NedbDataStore.ts

@@ -91,9 +91,15 @@ export class NeDBDataStore implements DataStore {
     public async setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise<void> {
         this.serverMappings[server.domain] = server;
 
-        for (const channel of Object.keys(serverConfig.mappings)) {
+        await this.removeConfigMappings(server);
+
+        for (const [channel, opts] of Object.entries(serverConfig.mappings)) {
+            if (opts.createRoom) {
+                return;
+            }
+            log.info(`Mapping channel ${channel} to ${opts.roomIds.join(",")}`);
             const ircRoom = new IrcRoom(server, channel);
-            for (const roomId of serverConfig.mappings[channel].roomIds) {
+            for (const roomId of opts.roomIds) {
                 const mxRoom = new MatrixRoom(roomId);
                 await this.storeRoom(ircRoom, mxRoom, "config");
             }
@@ -413,13 +419,22 @@ export class NeDBDataStore implements DataStore {
         }).filter((e) => e !== "");
     }
 
-    public async removeConfigMappings() {
+    public async removeConfigMappings(server: IrcServer) {
         await this.roomStore.removeEntriesByLinkData({
             from_config: true // for backwards compatibility
         });
-        await this.roomStore.removeEntriesByLinkData({
-            origin: 'config'
+        // Filter for config entries which are from this network and are NOT autocreated.
+        let entries = await this.roomStore.getEntriesByLinkData({
+            origin: 'config',
         });
+        const notChannels = server.getAutoCreateMappings().map((c) => c.channel);
+        entries = (await entries).filter((e) => e.remote?.get("domain") === server.domain &&
+            !notChannels.includes(e.remote?.get("channel") as string));
+
+        // Remove just these entries.
+        for (const entry of entries) {
+            await this.roomStore.delete({id: entry.id, "data.origin": "config"});
+        }
     }
 
     public async getIpv6Counter(): Promise<number> {

src/datastore/postgres/PgDataStore.ts

@@ -63,10 +63,19 @@ export class PgDataStore implements DataStore {
     public async setServerFromConfig(server: IrcServer, serverConfig: IrcServerConfig): Promise<void> {
         this.serverMappings[server.domain] = server;
 
-        for (const channel of Object.keys(serverConfig.mappings)) {
+        await this.removeConfigMappings(server);
+
+        for (const [channel, data] of Object.entries(serverConfig.mappings)) {
+            if (data.createRoom) {
+                // We don't want to map this.
+                return;
+            }
+            if (!data.roomIds) {
+                throw Error(`roomIds not given on ${channel} config entry`);
+            }
             const ircRoom = new IrcRoom(server, channel);
             ircRoom.set("type", "channel");
-            for (const roomId of serverConfig.mappings[channel].roomIds) {
+            for (const roomId of data.roomIds) {
                 const mxRoom = new MatrixRoom(roomId);
                 await this.storeRoom(ircRoom, mxRoom, "config");
             }
@@ -357,8 +366,25 @@ export class PgDataStore implements DataStore {
         ).rows.map((e) => e.room_id);
     }
 
-    public async removeConfigMappings(): Promise<void> {
-        await this.pgPool.query("DELETE FROM rooms WHERE origin = 'config'");
+    public async removeConfigMappings(server: IrcServer): Promise<void> {
+        // We need to remove config mappings on startup so we can ensure what's in the config
+        // matches what's in the database. The problem is that autogenerated rooms should not
+        // be deleted as they will not be regenerated on startup.
+        //
+        // However, we *should* delete any of them that are no longer in the config, hence the
+        // gnarly code below.
+        let exclusionQuerys = [];
+        console.log(server.getNetworkId(), server.getAutoCreateMappings());
+        for (const mapping of server.getAutoCreateMappings()) {
+            exclusionQuerys.push(mapping.channel);
+        }
+        let query = `DELETE FROM rooms WHERE origin = 'config' AND irc_domain = $1`
+        if (exclusionQuerys.length) {
+            query += ` AND irc_channel NOT IN ('${exclusionQuerys.join("','")}')`;
+        }
+        this.pgPool.query(query, [
+            server.domain,
+        ]);
     }
 
     public async getIpv6Counter(): Promise<number> {

src/irc/IrcServer.ts

@@ -174,6 +174,12 @@ export class IrcServer {
         return Array.from(roomIds.keys());
     }
 
+    public getAutoCreateMappings() {
+        return Object.entries(this.config.mappings)
+            .filter(([, v]) => v.createRoom)
+            .map(([k, v]) => ({ channel: k, key: v.key }));
+    }
+
     public getChannelKey(channel: string) {
         return this.config.mappings[channel]?.key;
     }
@@ -661,6 +667,7 @@ export interface IrcServerConfig {
     mappings: {
         [channel: string]: {
             roomIds: string[];
+            createRoom?: boolean;
             key?: string;
         };
     };