feat(ElasticSearch): Postgres synchronization

config/custom-environment-variables.json

@@ -1,11 +1,16 @@
 {
   "port": "PORT",
   "env": "OC_ENV",
+  "services": {
+    "server": "ENABLE_SERVICE_SERVER",
+    "searchSync": "ENABLE_SERVICE_SEARCH_SYNC"
+  },
   "mailpit": {
     "client": "MAILPIT_CLIENT"
   },
   "elasticSearch": {
-    "url": "ELASTICSEARCH_URL"
+    "url": "ELASTICSEARCH_URL",
+    "maxSyncDelay": "ELASTICSEARCH_MAX_SYNC_DELAY"
   },
   "database": {
     "url": "PG_URL",

config/default.json

@@ -1,5 +1,9 @@
 {
   "port": "3060",
+  "services": {
+    "server": true,
+    "searchSync": false
+  },
   "mailpit": {
     "client": false
   },
@@ -16,6 +20,9 @@
     },
     "readOnly": false
   },
+  "elasticSearch": {
+    "maxSyncDelay": 5000
+  },
   "maintenancedb": {
     "url": "postgres://127.0.0.1:5432/postgres"
   },

config/e2e.json

@@ -2,6 +2,9 @@
   "database": {
     "url": "postgres://[email protected]:5432/opencollective_e2e"
   },
+  "services": {
+    "searchSync": false
+  },
   "mailpit": {
     "client": true
   },

config/test.json

@@ -1,5 +1,9 @@
 {
   "port": "3061",
+  "services": {
+    "server": true,
+    "searchSync": false
+  },
   "database": {
     "url": "postgres://[email protected]:5432/opencollective_test"
   },

package.json

@@ -116,6 +116,7 @@
     "pg": "8.13.1",
     "pg-connection-string": "2.7.0",
     "pg-format": "1.0.4",
+    "pg-listen": "1.7.0",
     "plaid": "29.0.0",
     "prepend-http": "3.0.1",
     "redis": "4.6.6",

server/index.js

@@ -7,14 +7,16 @@ import config from 'config';
 import express from 'express';
 import throng from 'throng';
 
+import { startElasticSearchPostgresSync, stopElasticSearchPostgresSync } from './lib/elastic-search/sync-postgres';
 import expressLib from './lib/express';
 import logger from './lib/logger';
 import { updateCachedFidoMetadata } from './lib/two-factor-authentication/fido-metadata';
+import { parseToBoolean } from './lib/utils';
 import routes from './routes';
 
 const workers = process.env.WEB_CONCURRENCY || 1;
 
-async function start(i) {
+async function startExpressServer(workerId) {
   const expressApp = express();
 
   await updateCachedFidoMetadata();
@@ -35,7 +37,7 @@ async function start(i) {
       host,
       server.address().port,
       config.env,
-      i,
+      workerId,
     );
   });
 
@@ -45,15 +47,49 @@ async function start(i) {
   return expressApp;
 }
 
-let app;
+// Start the express server
+let appPromise;
+if (parseToBoolean(config.services.server)) {
+  if (['production', 'staging'].includes(config.env) && workers > 1) {
+    throng({ worker: startExpressServer, count: workers }); // TODO: Thong is not compatible with the shutdown logic below
+  } else {
+    appPromise = startExpressServer(1);
+  }
+}
 
-if (['production', 'staging'].includes(config.env) && workers > 1) {
-  throng({ worker: start, count: workers });
-} else {
-  app = start(1);
+// Start the search sync job
+if (parseToBoolean(config.services.searchSync)) {
+  startElasticSearchPostgresSync();
 }
 
+let isShuttingDown = false;
+const gracefullyShutdown = async signal => {
+  if (!isShuttingDown) {
+    logger.info(`Received ${signal}. Shutting down.`);
+    isShuttingDown = true;
+
+    if (appPromise) {
+      await appPromise.then(app => {
+        if (app.__server__) {
+          logger.info('Closing express server');
+          app.__server__.close();
+        }
+      });
+    }
+
+    if (parseToBoolean(config.services.searchSync)) {
+      await stopElasticSearchPostgresSync();
+    }
+
+    process.exit();
+  }
+};
+
+process.on('exit', () => gracefullyShutdown('exit'));
+process.on('SIGINT', () => gracefullyShutdown('SIGINT'));
+process.on('SIGTERM', () => gracefullyShutdown('SIGTERM'));
+
 // This is used by tests
-export default async function () {
-  return app ? app : start(1);
+export default async function startServerForTest() {
+  return appPromise ?? startExpressServer(1);
 }

server/lib/elastic-search/adapters/ElasticSearchCollectivesAdapter.ts

@@ -2,10 +2,9 @@ import models, { Op } from '../../../models';
 import { stripHTMLOrEmpty } from '../../sanitize-html';
 import { ElasticSearchIndexName } from '../constants';
 
-import { ElasticSearchModelAdapter } from './ElasticSearchModelAdapter';
+import { ElasticSearchModelAdapter, FindEntriesToIndexOptions } from './ElasticSearchModelAdapter';
 
 export class ElasticSearchCollectivesAdapter implements ElasticSearchModelAdapter {
-  public readonly model = models.Collective;
   public readonly index = ElasticSearchIndexName.COLLECTIVES;
   public readonly mappings = {
     properties: {
@@ -30,15 +29,11 @@ export class ElasticSearchCollectivesAdapter implements ElasticSearchModelAdapte
     },
   } as const;
 
-  public async findEntriesToIndex(
-    options: {
-      offset?: number;
-      limit?: number;
-      fromDate?: Date;
-      maxId?: number;
-      ids?: number[];
-    } = {},
-  ): Promise<Array<InstanceType<typeof models.Collective>>> {
+  public getModel() {
+    return models.Collective;
+  }
+
+  public async findEntriesToIndex(options: FindEntriesToIndexOptions = {}) {
     return models.Collective.findAll({
       attributes: Object.keys(this.mappings.properties),
       order: [['id', 'DESC']],
@@ -49,6 +44,15 @@ export class ElasticSearchCollectivesAdapter implements ElasticSearchModelAdapte
         ...(options.fromDate ? { updatedAt: options.fromDate } : null),
         ...(options.maxId ? { id: { [Op.lte]: options.maxId } } : null),
         ...(options.ids?.length ? { id: { [Op.in]: options.ids } } : null),
+        ...(options.relatedToCollectiveIds?.length
+          ? {
+              [Op.or]: [
+                { id: options.relatedToCollectiveIds },
+                { HostCollectiveId: options.relatedToCollectiveIds },
+                { ParentCollectiveId: options.relatedToCollectiveIds },
+              ],
+            }
+          : null),
       },
     });
   }
@@ -72,7 +76,7 @@ export class ElasticSearchCollectivesAdapter implements ElasticSearchModelAdapte
       isActive: instance.isActive,
       isHostAccount: instance.isHostAccount,
       deactivatedAt: instance.deactivatedAt,
-      HostCollectiveId: instance.HostCollectiveId,
+      HostCollectiveId: !instance.isActive ? null : instance.HostCollectiveId,
       ParentCollectiveId: instance.ParentCollectiveId,
     };
   }

server/lib/elastic-search/adapters/ElasticSearchCommentsAdapter.ts

@@ -7,10 +7,9 @@ import { CommentType } from '../../../models/Comment';
 import { stripHTMLOrEmpty } from '../../sanitize-html';
 import { ElasticSearchIndexName } from '../constants';
 
-import { ElasticSearchModelAdapter } from './ElasticSearchModelAdapter';
+import { ElasticSearchModelAdapter, FindEntriesToIndexOptions } from './ElasticSearchModelAdapter';
 
 export class ElasticSearchCommentsAdapter implements ElasticSearchModelAdapter {
-  public readonly model = models.Comment;
   public readonly index = ElasticSearchIndexName.COMMENTS;
   public readonly mappings = {
     properties: {
@@ -29,15 +28,11 @@ export class ElasticSearchCommentsAdapter implements ElasticSearchModelAdapter {
     },
   } as const;
 
-  public findEntriesToIndex(
-    options: {
-      offset?: number;
-      limit?: number;
-      fromDate?: Date;
-      maxId?: number;
-      ids?: number[];
-    } = {},
-  ) {
+  public getModel() {
+    return models.Comment;
+  }
+
+  public findEntriesToIndex(options: FindEntriesToIndexOptions = {}) {
     return models.Comment.findAll({
       attributes: omit(Object.keys(this.mappings.properties), ['HostCollectiveId', 'ParentCollectiveId']),
       order: [['id', 'DESC']],
@@ -47,12 +42,20 @@ export class ElasticSearchCommentsAdapter implements ElasticSearchModelAdapter {
         ...(options.fromDate ? { updatedAt: options.fromDate } : null),
         ...(options.maxId ? { id: { [Op.lte]: options.maxId } } : null),
         ...(options.ids?.length ? { id: { [Op.in]: options.ids } } : null),
+        ...(options.relatedToCollectiveIds?.length
+          ? {
+              [Op.or]: [
+                { CollectiveId: options.relatedToCollectiveIds },
+                { FromCollectiveId: options.relatedToCollectiveIds },
+              ],
+            }
+          : null),
       },
       include: [
         {
           association: 'collective',
           required: true,
-          attributes: ['HostCollectiveId', 'ParentCollectiveId'],
+          attributes: ['isActive', 'HostCollectiveId', 'ParentCollectiveId'],
         },
         {
           association: 'expense',
@@ -84,7 +87,7 @@ export class ElasticSearchCommentsAdapter implements ElasticSearchModelAdapter {
       HostCollectiveId:
         instance.expense?.HostCollectiveId ??
         instance.hostApplication?.HostCollectiveId ??
-        instance.collective.HostCollectiveId,
+        (!instance.collective.isActive ? null : instance.collective.HostCollectiveId),
     };
   }
 

server/lib/elastic-search/adapters/ElasticSearchExpensesAdapter.ts

@@ -5,10 +5,9 @@ import models from '../../../models';
 import { stripHTMLOrEmpty } from '../../sanitize-html';
 import { ElasticSearchIndexName } from '../constants';
 
-import { ElasticSearchModelAdapter } from './ElasticSearchModelAdapter';
+import { ElasticSearchModelAdapter, FindEntriesToIndexOptions } from './ElasticSearchModelAdapter';
 
 export class ElasticSearchExpensesAdapter implements ElasticSearchModelAdapter {
-  public readonly model = models.Expense;
   public readonly index = ElasticSearchIndexName.EXPENSES;
   public readonly mappings = {
     properties: {
@@ -34,15 +33,11 @@ export class ElasticSearchExpensesAdapter implements ElasticSearchModelAdapter {
     },
   } as const;
 
-  public findEntriesToIndex(
-    options: {
-      offset?: number;
-      limit?: number;
-      fromDate?: Date;
-      maxId?: number;
-      ids?: number[];
-    } = {},
-  ) {
+  public getModel() {
+    return models.Expense;
+  }
+
+  public findEntriesToIndex(options: FindEntriesToIndexOptions = {}) {
     return models.Expense.findAll({
       attributes: omit(Object.keys(this.mappings.properties), ['ParentCollectiveId', 'items']),
       order: [['id', 'DESC']],
@@ -51,13 +46,27 @@ export class ElasticSearchExpensesAdapter implements ElasticSearchModelAdapter {
       where: {
         ...(options.fromDate ? { updatedAt: options.fromDate } : null),
         ...(options.maxId ? { id: { [Op.lte]: options.maxId } } : null),
-        ...(options.ids?.length ? { id: { [Op.in]: options.ids } } : null),
+        ...(options.ids?.length ? { id: options.ids } : null),
+        ...(options.relatedToCollectiveIds?.length
+          ? {
+              [Op.or]: [
+                { CollectiveId: options.relatedToCollectiveIds },
+                { FromCollectiveId: options.relatedToCollectiveIds },
+                { HostCollectiveId: options.relatedToCollectiveIds },
+              ],
+            }
+          : null),
       },
       include: [
         {
           association: 'collective',
           required: true,
-          attributes: ['HostCollectiveId', 'ParentCollectiveId'],
+          attributes: ['isActive', 'HostCollectiveId', 'ParentCollectiveId'],
+        },
+        {
+          association: 'items',
+          required: true,
+          attributes: ['description'],
         },
         {
           association: 'items',
@@ -87,8 +96,9 @@ export class ElasticSearchExpensesAdapter implements ElasticSearchModelAdapter {
       ParentCollectiveId: instance.collective.ParentCollectiveId,
       FromCollectiveId: instance.FromCollectiveId,
       UserId: instance.UserId,
-      HostCollectiveId: instance.HostCollectiveId || instance.collective.HostCollectiveId,
-      items: instance.items.map(item => item.description).join(' '),
+      items: instance.items.map(item => item.description).join(', '),
+      HostCollectiveId:
+        instance.HostCollectiveId || (!instance.collective.isActive ? null : instance.collective.HostCollectiveId),
     };
   }