From 30578fc83774269b52bb0bd3d51a4cce8ca4f6cb Mon Sep 17 00:00:00 2001 From: Max Stoiber Date: Tue, 4 Jun 2019 14:26:22 +0200 Subject: [PATCH] Implement offsite backups --- chronos/index.js | 3 ++ chronos/jobs/index.js | 15 ++++++-- chronos/queues/constants.js | 1 + chronos/queues/database-backup.js | 17 ++------- chronos/queues/offsite-backup.js | 61 +++++++++++++++++++++++++++++++ chronos/utils/compose.js | 20 ++++++++++ shared/bull/queues.js | 2 + shared/bull/types.js | 1 + 8 files changed, 102 insertions(+), 18 deletions(-) create mode 100644 chronos/queues/offsite-backup.js create mode 100644 chronos/utils/compose.js diff --git a/chronos/index.js b/chronos/index.js index 1b1af6e8d2..5df60ea539 100644 --- a/chronos/index.js +++ b/chronos/index.js @@ -9,6 +9,7 @@ import processDailyCoreMetrics from 'chronos/queues/coreMetrics'; import processActiveCommunityAdminReport from 'chronos/queues/coreMetrics/activeCommunityAdminReport'; import processRemoveSeenUsersNotifications from 'chronos/queues/remove-seen-usersNotifications'; import processDatabaseBackup from 'chronos/queues/database-backup'; +import processOffsiteBackup from 'chronos/queues/offsite-backup'; import { PROCESS_WEEKLY_DIGEST_EMAIL, PROCESS_DAILY_DIGEST_EMAIL, @@ -17,6 +18,7 @@ import { PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT, PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS, PROCESS_DATABASE_BACKUP, + PROCESS_OFFSITE_BACKUP, } from 'chronos/queues/constants'; import { startJobs } from 'chronos/jobs'; @@ -34,6 +36,7 @@ const server = createWorker( [PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT]: processActiveCommunityAdminReport, [PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS]: processRemoveSeenUsersNotifications, [PROCESS_DATABASE_BACKUP]: processDatabaseBackup, + [PROCESS_OFFSITE_BACKUP]: processOffsiteBackup, }, { settings: { diff --git a/chronos/jobs/index.js b/chronos/jobs/index.js index a1a0ed2b21..3f6c3a07a7 100644 --- a/chronos/jobs/index.js +++ b/chronos/jobs/index.js @@ -8,6 +8,7 @@ import { activeCommunityReportQueue, removeSeenUsersNotificationsQueue, databaseBackupQueue, + offsiteBackupQueue, } from 'shared/bull/queues'; /* @@ -38,9 +39,14 @@ export const activeCommunityReport = () => { ); }; -export const dailyBackups = () => { - // at 9am every day (~12 hours away from the automatic daily backup) - return databaseBackupQueue.add(undefined, defaultJobOptions('0 9 * * *')); +export const hourlyBackups = () => { + // Every hour + return databaseBackupQueue.add(undefined, defaultJobOptions('30 * * * *')); +}; + +export const hourlyOffsiteBackup = () => { + // Every hour offset by 30m from hourly backups, which should be enough time for the backups to finish + return offsiteBackupQueue.add(undefined, defaultJobOptions('0 * * * *')); }; export const removeSeenUsersNotifications = () => { @@ -57,5 +63,6 @@ export const startJobs = () => { dailyCoreMetrics(); activeCommunityReport(); removeSeenUsersNotifications(); - dailyBackups(); + hourlyBackups(); + hourlyOffsiteBackup(); }; diff --git a/chronos/queues/constants.js b/chronos/queues/constants.js index f0596c2d1f..4430e794a8 100644 --- a/chronos/queues/constants.js +++ b/chronos/queues/constants.js @@ -27,3 +27,4 @@ export const PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT = export const PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS = 'process remove seen usersNotifications'; export const PROCESS_DATABASE_BACKUP = 'process database backup'; +export const PROCESS_OFFSITE_BACKUP = 'process offsite backup'; diff --git a/chronos/queues/database-backup.js b/chronos/queues/database-backup.js index 6026ad85b2..9b63e0425c 100644 --- a/chronos/queues/database-backup.js +++ b/chronos/queues/database-backup.js @@ -1,24 +1,13 @@ // @flow const debug = require('debug')('chronos:queue:database-backup'); -const fetch = require('node-fetch'); +const { compose, COMPOSE_DEPLOYMENT_ID } = require('../utils/compose'); -const COMPOSE_API_TOKEN = process.env.COMPOSE_API_TOKEN; const processJob = async () => { - if (!COMPOSE_API_TOKEN) { - console.warn( - 'Cannot start hourly backup, COMPOSE_API_TOKEN env variable is missing.' - ); - return; - } debug('pinging compose to start on-demand db backup'); - const result = await fetch( - 'https://api.compose.io/2016-07/deployments/5cd38bcf9c5cab000b617356/backups', + const result = await compose( + `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups`, { method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${COMPOSE_API_TOKEN}`, - }, } ); const json = await result.json(); diff --git a/chronos/queues/offsite-backup.js b/chronos/queues/offsite-backup.js new file mode 100644 index 0000000000..76a51bbc3a --- /dev/null +++ b/chronos/queues/offsite-backup.js @@ -0,0 +1,61 @@ +// @flow +import AWS from 'aws-sdk'; +const https = require('https'); +const { compose, COMPOSE_DEPLOYMENT_ID } = require('../utils/compose'); + +AWS.config.update({ + accessKeyId: process.env.S3_TOKEN || 'asdf123', + secretAccessKey: process.env.S3_SECRET || 'asdf123', + apiVersions: { + s3: 'latest', + }, +}); +const s3 = new AWS.S3(); + +export default async () => { + const backupListResult = await compose( + `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups` + ); + + const backupListJson = await backupListResult.json(); + + if (!backupListJson._embedded || !backupListJson._embedded.backups) { + console.error( + `Failed to load list of backups of deployment ${COMPOSE_DEPLOYMENT_ID}.` + ); + return; + } + + const newestBackup = backupListJson._embedded.backups + .filter(backup => backup.is_downloadable) + .sort((a, b) => new Date(b.created_at) - new Date(a.created_at))[0]; + + if (!newestBackup) { + console.error('Failed to find latest backup.'); + return; + } + + const backupResult = await compose( + `2016-07/deployments/${COMPOSE_DEPLOYMENT_ID}/backups/${newestBackup.id}` + ); + const backupJson = await backupResult.json(); + + await new Promise((resolve, reject) => { + https.get(backupJson.download_link, response => { + s3.upload( + { + Body: response, + Bucket: `spectrum-chat/backups`, + Key: backupJson.name, + }, + function(err) { + if (err) { + console.error(err); + return reject(err); + } + return resolve(); + } + ); + }); + }); +}; diff --git a/chronos/utils/compose.js b/chronos/utils/compose.js new file mode 100644 index 0000000000..e66fb9536a --- /dev/null +++ b/chronos/utils/compose.js @@ -0,0 +1,20 @@ +// @flow +import fetch from 'node-fetch'; + +const COMPOSE_API_TOKEN = process.env.COMPOSE_API_TOKEN; + +export const COMPOSE_DEPLOYMENT_ID = '5cd38bcf9c5cab000b617356'; + +export const compose = (path: string, fetchOptions?: Object = {}) => { + if (!COMPOSE_API_TOKEN) { + throw new Error('Please specify the COMPOSE_API_TOKEN env var.'); + } + return fetch(`https://api.compose.io/${path}`, { + ...fetchOptions, + headers: { + 'Content-Type': 'application/json', + ...(fetchOptions.headers || {}), + Authorization: `Bearer ${COMPOSE_API_TOKEN}`, + }, + }); +}; diff --git a/shared/bull/queues.js b/shared/bull/queues.js index 1c052929fd..b5935669db 100644 --- a/shared/bull/queues.js +++ b/shared/bull/queues.js @@ -71,6 +71,7 @@ import { PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT, PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS, PROCESS_DATABASE_BACKUP, + PROCESS_OFFSITE_BACKUP, } from 'chronos/queues/constants'; // Normalize our (inconsistent) queue names to a set of JS compatible names @@ -143,6 +144,7 @@ exports.QUEUE_NAMES = { activeCommunityReportQueue: PROCESS_ACTIVE_COMMUNITY_ADMIN_REPORT, removeSeenUsersNotificationsQueue: PROCESS_REMOVE_SEEN_USERS_NOTIFICATIONS, databaseBackupQueue: PROCESS_DATABASE_BACKUP, + offsiteBackupQueue: PROCESS_OFFSITE_BACKUP, }; // We add one error listener per queue, so we have to set the max listeners diff --git a/shared/bull/types.js b/shared/bull/types.js index 406e110624..17532018e2 100644 --- a/shared/bull/types.js +++ b/shared/bull/types.js @@ -523,4 +523,5 @@ export type Queues = { activeCommunityReportQueue: BullQueue, removeSeenUsersNotificationsQueue: BullQueue, databaseBackupQueue: BullQueue, + offsiteBackupQueue: BullQueue, };