Optimised SQL query

ghost/admin/app/controllers/members.js

@@ -3,6 +3,7 @@ import BulkDeleteMembersModal from '../components/members/modals/bulk-delete';
 import BulkRemoveMembersLabelModal from '../components/members/modals/bulk-remove-label';
 import BulkUnsubscribeMembersModal from '../components/members/modals/bulk-unsubscribe';
 import Controller from '@ember/controller';
+import fetch from 'fetch';
 import ghostPaths from 'ghost-admin/utils/ghost-paths';
 import moment from 'moment-timezone';
 import {A} from '@ember/array';
@@ -369,12 +370,24 @@ export default class MembersController extends Controller {
     }
 
     @action
-    exportData() {
+    async exportData() {
         let exportUrl = ghostPaths().url.api('members/upload');
         let downloadParams = new URLSearchParams(this.getApiQueryObject());
         downloadParams.set('limit', 'all');
 
-        this.utils.downloadFile(`${exportUrl}?${downloadParams.toString()}`);
+        const response = await fetch(`${exportUrl}?${downloadParams.toString()}`);
+        const data = await response.json();
+
+        if (data.url) {
+            const link = document.createElement('a');
+            link.href = data.url;
+            const datetime = (new Date()).toJSON().substring(0, 10);
+            link.download = `members.${datetime}.csv`;
+
+            document.body.appendChild(link);
+            link.click();
+            document.body.removeChild(link);
+        }
     }
 
     @action

ghost/core/core/server/api/endpoints/members.js

@@ -374,13 +374,6 @@ const controller = {
             'search'
         ],
         headers: {
-            disposition: {
-                type: 'csv',
-                value() {
-                    const datetime = (new Date()).toJSON().substring(0, 10);
-                    return `members.${datetime}.csv`;
-                }
-            },
             cacheInvalidate: false
         },
         response: {
@@ -391,8 +384,9 @@ const controller = {
         },
         validation: {},
         async query(frame) {
+            const memberFileUrl = await membersService.export(frame.options);
             return {
-                data: await membersService.export(frame.options)
+                url: memberFileUrl
             };
         }
     },

ghost/core/core/server/api/endpoints/utils/serializers/output/members.js

@@ -1,7 +1,6 @@
 //@ts-check
 const _ = require('lodash');
 const debug = require('@tryghost/debug')('api:endpoints:utils:serializers:output:members');
-const {unparse} = require('@tryghost/members-csv');
 const mappers = require('./mappers');
 
 module.exports = {
@@ -91,11 +90,10 @@ function activityFeed(data, _apiConfig, frame) {
  *
  * @param {{data: any[]}} data
  *
- * @returns {string} - A CSV string
+ * @returns {object} - A CSV string
  */
 function exportCSV(data) {
-    debug('exportCSV');
-    return unparse(data.data);
+    return data;
 }
 
 function serializeAttribution(attribution) {

ghost/core/core/server/services/members/exporter/query.js

@@ -1,6 +1,11 @@
 const models = require('../../../models');
 const {knex} = require('../../../data/db');
 const moment = require('moment');
+const {unparse} = require('@tryghost/members-csv');
+const urlUtils = require('../../../../shared/url-utils');
+const storage = require('../../../adapters/storage');
+const crypto = require('crypto');
+const glob = require('glob');
 
 module.exports = async function (options) {
     const hasFilter = options.limit !== 'all' || options.filter || options.search;
@@ -9,6 +14,7 @@ module.exports = async function (options) {
     if (hasFilter) {
         // do a very minimal query, only to fetch the ids of the filtered values
         // should be quite fast
+
         options.withRelated = [];
         options.columns = ['id'];
 
@@ -34,55 +40,143 @@ module.exports = async function (options) {
     const allProducts = await models.Product.fetchAll();
     const allLabels = await models.Label.fetchAll();
 
-    let query = knex('members')
-        .select('id', 'email', 'name', 'note', 'status', 'created_at')
-        .select(knex.raw(`
-            (CASE WHEN EXISTS (SELECT 1 FROM members_newsletters n WHERE n.member_id = members.id)
-                    THEN TRUE ELSE FALSE
-            END) as subscribed
-        `))
-        .select(knex.raw(`
-            (SELECT GROUP_CONCAT(product_id) FROM members_products f WHERE f.member_id = members.id) as tiers
-        `))
-        .select(knex.raw(`
-            (SELECT GROUP_CONCAT(label_id) FROM members_labels f WHERE f.member_id = members.id) as labels
-        `))
-        .select(knex.raw(`
-            (SELECT customer_id FROM members_stripe_customers f WHERE f.member_id = members.id limit 1) as stripe_customer_id
-        `));
+    /*
 
-    if (hasFilter) {
-        query = query.whereIn('id', ids);
-    }
+    explain analyze SELECT 
+    id,
+    email,
+    name,
+    note,
+    status,
+    created_at,
+    CASE 
+        WHEN EXISTS (
+            SELECT 1 
+            FROM members_newsletters n 
+            WHERE n.member_id = members.id
+        ) THEN TRUE 
+        ELSE FALSE 
+    END AS subscribed,
+    (SELECT GROUP_CONCAT(product_id) 
+    FROM members_products f 
+    WHERE f.member_id = members.id) AS tiers,
+    (SELECT GROUP_CONCAT(label_id) 
+    FROM members_labels f 
+    WHERE f.member_id = members.id) AS labels,
+    (SELECT customer_id 
+    FROM members_stripe_customers f 
+    WHERE f.member_id = members.id LIMIT 1) AS stripe_customer_id
+    FROM members
+
+    */
+
+    /*
+    explain analyze SELECT id, email, name, note, status, created_at
+    FROM members;
+
+    explain analyze SELECT member_id, GROUP_CONCAT(product_id) AS tiers
+    FROM members_products
+    GROUP BY member_id;
+
+    explain analyze SELECT member_id, GROUP_CONCAT(label_id) AS labels
+    FROM members_labels
+    GROUP BY member_id;
 
-    const rows = await query;
-    for (const row of rows) {
-        const tierIds = row.tiers ? row.tiers.split(',') : [];
-        const tiers = tierIds.map((id) => {
+    explain analyze SELECT member_id, MIN(customer_id) AS stripe_customer_id
+    FROM members_stripe_customers
+    GROUP BY member_id;
+
+    explain analyze SELECT DISTINCT member_id
+    FROM members_newsletters;
+    */
+    const [members, tiers, labels, stripeCustomers, subscriptions] = await Promise.all([
+        knex('members')
+            .select('id', 'email', 'name', 'note', 'status', 'created_at')
+            .modify((query) => {
+                if (hasFilter) {
+                    query.whereIn('id', ids);
+                }
+            }),
+        knex('members_products')
+            .select('member_id', knex.raw('GROUP_CONCAT(product_id) as tiers'))
+            .groupBy('member_id')
+            .modify((query) => {
+                if (hasFilter) {
+                    query.whereIn('member_id', ids);
+                }
+            }),
+        knex('members_labels')
+            .select('member_id', knex.raw('GROUP_CONCAT(label_id) as labels'))
+            .groupBy('member_id')
+            .modify((query) => {
+                if (hasFilter) {
+                    query.whereIn('member_id', ids);
+                }
+            }),
+        knex('members_stripe_customers')
+            .select('member_id', knex.raw('MIN(customer_id) as stripe_customer_id'))
+            .groupBy('member_id')
+            .modify((query) => {
+                if (hasFilter) {
+                    query.whereIn('member_id', ids);
+                }
+            }),
+        knex('members_newsletters')
+            .distinct('member_id')
+            .modify((query) => {
+                if (hasFilter) {
+                    query.whereIn('member_id', ids);
+                }
+            })
+    ]);
+
+    const tiersMap = new Map(tiers.map(row => [row.member_id, row.tiers]));
+    const labelsMap = new Map(labels.map(row => [row.member_id, row.labels]));
+    const stripeCustomerMap = new Map(stripeCustomers.map(row => [row.member_id, row.stripe_customer_id]));
+    const subscribedSet = new Set(subscriptions.map(row => row.member_id));
+
+    for (const row of members) {
+        const tierIds = tiersMap.get(row.id) ? tiersMap.get(row.id).split(',') : [];
+        const tierDetails = tierIds.map((id) => {
             const tier = allProducts.find(p => p.id === id);
             return {
                 name: tier.get('name')
             };
         });
-        row.tiers = tiers;
+        row.tiers = tierDetails;
 
-        const labelIds = row.labels ? row.labels.split(',') : [];
-        const labels = labelIds.map((id) => {
+        const labelIds = labelsMap.get(row.id) ? labelsMap.get(row.id).split(',') : [];
+        const labelDetails = labelIds.map((id) => {
             const label = allLabels.find(l => l.id === id);
             return {
                 name: label.get('name')
             };
         });
-        row.labels = labels;
-    }
+        row.labels = labelDetails;
 
-    for (const member of rows) {
-        // Note: we don't modify the array or change/duplicate objects
-        // to increase performance
-        member.subscribed = !!member.subscribed;
-        member.comped = member.status === 'comped';
-        member.created_at = moment(member.created_at).toISOString();
+        row.subscribed = subscribedSet.has(row.id);
+        row.comped = row.status === 'comped';
+        row.stripe_customer_id = stripeCustomerMap.get(row.id) || null;
+        row.created_at = moment(row.created_at).toISOString();
     }
 
-    return rows;
+    const csv = unparse(members);
+
+    return await saveCsv(csv);
 };
+
+async function saveCsv(csv) {
+    const store = storage.getStorage('files');
+    const files = glob.sync('content/files/members/members*');
+    files.forEach((file) => {
+        store.delete(file, ''); // delete all previously generated member csv files to free up storage space
+    });
+    const csvStoredUrl = await store.saveRaw(csv, `/members/${generateUniqueFileName()}`);
+    return `${urlUtils.urlFor('files', true)}${csvStoredUrl.replace(/^\/+/, '')}`;
+}
+
+function generateUniqueFileName() {
+    const timestamp = new Date().toISOString();
+    const randomHash = crypto.randomBytes(16).toString('hex');
+    return `members-${timestamp}-${randomHash}.csv`;
+}