Merge pull request #5150 from withspectrum/off-site-backups

Implement offsite backups

chronos/index.js

@@ -9,6 +9,7 @@ import processDailyCoreMetrics from 'chronos/queues/coreMetrics';
 import processActiveCommunityAdminReport from 'chronos/queues/coreMetrics/activeCommunityAdminReport';
 import processRemoveSeenUsersNotifications from 'chronos/queues/remove-seen-usersNotifications';
 import processDatabaseBackup from 'chronos/queues/database-backup';
+import processOffsiteBackup from 'chronos/queues/offsite-backup';
 import {
   PROCESS_WEEKLY_DIGEST_EMAIL,
   PROCESS_DAILY_DIGEST_EMAIL,
@@ -17,6 +18,7 @@ import {
   PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT,
   PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS,
   PROCESS_DATABASE_BACKUP,
+  PROCESS_OFFSITE_BACKUP,
 } from 'chronos/queues/constants';
 import { startJobs } from 'chronos/jobs';
 
@@ -34,6 +36,7 @@ const server = createWorker(
     [PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT]: processActiveCommunityAdminReport,
     [PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS]: processRemoveSeenUsersNotifications,
     [PROCESS_DATABASE_BACKUP]: processDatabaseBackup,
+    [PROCESS_OFFSITE_BACKUP]: processOffsiteBackup,
   },
   {
     settings: {

chronos/jobs/index.js

@@ -8,6 +8,7 @@ import {
   activeCommunityReportQueue,
   removeSeenUsersNotificationsQueue,
   databaseBackupQueue,
+  offsiteBackupQueue,
 } from 'shared/bull/queues';
 
 /*
@@ -38,9 +39,14 @@ export const activeCommunityReport = () => {
   );
 };
 
-export const dailyBackups = () => {
-  // at 9am every day (~12 hours away from the automatic daily backup)
-  return databaseBackupQueue.add(undefined, defaultJobOptions('0 9 * * *'));
+export const hourlyBackups = () => {
+  // Every hour
+  return databaseBackupQueue.add(undefined, defaultJobOptions('30 * * * *'));
+};
+
+export const hourlyOffsiteBackup = () => {
+  // Every hour offset by 30m from hourly backups, which should be enough time for the backups to finish
+  return offsiteBackupQueue.add(undefined, defaultJobOptions('0 * * * *'));
 };
 
 export const removeSeenUsersNotifications = () => {
@@ -57,5 +63,6 @@ export const startJobs = () => {
   dailyCoreMetrics();
   activeCommunityReport();
   removeSeenUsersNotifications();
-  dailyBackups();
+  hourlyBackups();
+  hourlyOffsiteBackup();
 };

chronos/queues/constants.js

@@ -27,3 +27,4 @@ export const PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT =
 export const PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS =
   'process remove seen usersNotifications';
 export const PROCESS_DATABASE_BACKUP = 'process database backup';
+export const PROCESS_OFFSITE_BACKUP = 'process offsite backup';

chronos/queues/database-backup.js

@@ -1,24 +1,13 @@
 // @flow
 const debug = require('debug')('chronos:queue:database-backup');
-const fetch = require('node-fetch');
+const { compose, COMPOSE_DEPLOYMENT_ID } = require('../utils/compose');
 
-const COMPOSE_API_TOKEN = process.env.COMPOSE_API_TOKEN;
 const processJob = async () => {
-  if (!COMPOSE_API_TOKEN) {
-    console.warn(
-      'Cannot start hourly backup, COMPOSE_API_TOKEN env variable is missing.'
-    );
-    return;
-  }
   debug('pinging compose to start on-demand db backup');
-  const result = await fetch(
-    'https://api.compose.io/2016-07/deployments/5cd38bcf9c5cab000b617356/backups',
+  const result = await compose(
+    `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups`,
     {
       method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${COMPOSE_API_TOKEN}`,
-      },
     }
   );
   const json = await result.json();

chronos/queues/offsite-backup.js

@@ -0,0 +1,61 @@
+// @flow
+import AWS from 'aws-sdk';
+const https = require('https');
+const { compose, COMPOSE_DEPLOYMENT_ID } = require('../utils/compose');
+
+AWS.config.update({
+  accessKeyId: process.env.S3_TOKEN || 'asdf123',
+  secretAccessKey: process.env.S3_SECRET || 'asdf123',
+  apiVersions: {
+    s3: 'latest',
+  },
+});
+const s3 = new AWS.S3();
+
+export default async () => {
+  const backupListResult = await compose(
+    `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups`
+  );
+
+  const backupListJson = await backupListResult.json();
+
+  if (!backupListJson._embedded || !backupListJson._embedded.backups) {
+    console.error(
+      `Failed to load list of backups of deployment ${COMPOSE_DEPLOYMENT_ID}.`
+    );
+    return;
+  }
+
+  const newestBackup = backupListJson._embedded.backups
+    .filter(backup => backup.is_downloadable)
+    .sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0];
+
+  if (!newestBackup) {
+    console.error('Failed to find latest backup.');
+    return;
+  }
+
+  const backupResult = await compose(
+    `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups/${newestBackup.id}`
+  );
+  const backupJson = await backupResult.json();
+
+  await new Promise((resolve, reject) => {
+    https.get(backupJson.download_link, response => {
+      s3.upload(
+        {
+          Body: response,
+          Bucket: `spectrum-chat/backups`,
+          Key: backupJson.name,
+        },
+        function(err) {
+          if (err) {
+            console.error(err);
+            return reject(err);
+          }
+          return resolve();
+        }
+      );
+    });
+  });
+};

chronos/utils/compose.js

@@ -0,0 +1,20 @@
+// @flow
+import fetch from 'node-fetch';
+
+const COMPOSE_API_TOKEN = process.env.COMPOSE_API_TOKEN;
+
+export const COMPOSE_DEPLOYMENT_ID = '5cd38bcf9c5cab000b617356';
+
+export const compose = (path: string, fetchOptions?: Object = {}) => {
+  if (!COMPOSE_API_TOKEN) {
+    throw new Error('Please specify the COMPOSE_API_TOKEN env var.');
+  }
+  return fetch(`https://api.compose.io/${path}`, {
+    ...fetchOptions,
+    headers: {
+      'Content-Type': 'application/json',
+      ...(fetchOptions.headers || {}),
+      Authorization: `Bearer ${COMPOSE_API_TOKEN}`,
+    },
+  });
+};

shared/bull/queues.js

@@ -71,6 +71,7 @@ import {
   PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT,
   PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS,
   PROCESS_DATABASE_BACKUP,
+  PROCESS_OFFSITE_BACKUP,
 } from 'chronos/queues/constants';
 
 // Normalize our (inconsistent) queue names to a set of JS compatible names
@@ -143,6 +144,7 @@ exports.QUEUE_NAMES = {
   activeCommunityReportQueue: PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT,
   removeSeenUsersNotificationsQueue: PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS,
   databaseBackupQueue: PROCESS_DATABASE_BACKUP,
+  offsiteBackupQueue: PROCESS_OFFSITE_BACKUP,
 };
 
 // We add one error listener per queue, so we have to set the max listeners

shared/bull/types.js

@@ -523,4 +523,5 @@ export type Queues = {
   activeCommunityReportQueue: BullQueue<void>,
   removeSeenUsersNotificationsQueue: BullQueue<void>,
   databaseBackupQueue: BullQueue<void>,
+  offsiteBackupQueue: BullQueue<void>,
 };