fix: improve queries performance

snapshot-labs · Jul 18, 2024 · 5daf73f · 5daf73f
1 parent 36fff70
commit 5daf73f
Showing 1 changed file with 142 additions and 99 deletions.
diff --git a/src/helpers/spaces.ts b/src/helpers/spaces.ts
@@ -5,12 +5,36 @@ import db from './mysql';
 import log from './log';
 import { capture } from '@snapshot-labs/snapshot-sentry';
 
+const RUN_INTERVAL = 120e3;
+
 export let spaces = {};
-export const spacesMetadata = {};
-export let rankedSpaces: any = [];
-const spaceProposals = {};
-const spaceVotes = {};
-const spaceFollowers = {};
+export let rankedSpaces: Metadata[] = [];
+export const spacesMetadata: Record<string, Metadata> = {};
+
+type Metadata = {
+  id: string;
+  name: string;
+  verified: boolean;
+  flagged: boolean;
+  turbo: boolean;
+  hibernated: boolean;
+  popularity: number;
+  rank: number;
+  private: boolean;
+  categories: string[];
+  networks: string[];
+  counts: {
+    activeProposals: number;
+    proposalsCount: number;
+    proposalsCount7d: number;
+    followersCount: number;
+    followersCount7d: number;
+    votesCount: number;
+    votesCount7d: number;
+  };
+  strategyNames: string[];
+  pluginNames: string[];
+};
 
 const testnets = Object.values(networks)
   .filter((network: any) => network.testnet)
@@ -23,43 +47,45 @@ const testStrategies = [
   'api-v2-override'
 ];
 
-function getPopularity(
-  id: string,
-  params: {
-    verified: boolean;
-    turbo: boolean;
-    networks: string[];
-    strategies: any[];
-  }
-): number {
+function getPopularity(space: Metadata): number {
   let popularity =
-    (spaceVotes[id]?.count || 0) / 100 +
-    (spaceVotes[id]?.count_7d || 0) +
-    (spaceProposals[id]?.count || 0) / 100 +
-    (spaceProposals[id]?.count_7d || 0) +
-    (spaceFollowers[id]?.count || 0) / 50 +
-    (spaceFollowers[id]?.count_7d || 0);
-
-  if (params.networks.some(network => testnets.includes(network)))
+    space.counts.votesCount / 100 +
+    space.counts.votesCount7d +
+    space.counts.proposalsCount / 100 +
+    space.counts.proposalsCount7d +
+    space.counts.followersCount / 50 +
+    space.counts.followersCount7d;
+
+  if (space.networks.some(network => testnets.includes(network)))
     popularity = 1;
-  if (params.strategies.some(strategy => testStrategies.includes(strategy)))
+  if (space.strategyNames.some(strategy => testStrategies.includes(strategy)))
     popularity = 1;
 
-  if (params.verified) popularity *= 100000000;
-  if (params.turbo) {
+  if (space.verified) popularity *= 100000000;
+  if (space.turbo) {
     popularity += 1;
     popularity *= 100000000;
   }
 
   return popularity;
 }
 
+function sortSpaces() {
+  Object.entries(spacesMetadata).forEach(([id, space]: any) => {
+    spacesMetadata[id].popularity = getPopularity(space);
+  });
+
+  rankedSpaces = Object.values(spacesMetadata)
+    .filter((space: any) => !space.private && !space.flagged)
+    .sort((a: any, b: any) => b.popularity - a.popularity);
+
+  rankedSpaces.forEach((space, i) => {
+    spacesMetadata[space.id].rank = i + 1;
+  });
+}
+
 function mapSpaces() {
   Object.entries(spaces).forEach(([id, space]: any) => {
-    const verified = space.verified || false;
-    const flagged = space.flagged || false;
-    const turbo = space.turbo || false;
-    const hibernated = space.hibernated || false;
     const networks = uniq(
       (space.strategies || [])
         .map(strategy => strategy?.network || space.network)
@@ -69,60 +95,47 @@ function mapSpaces() {
       (space.strategies || []).map(strategy => strategy.name)
     );
     const pluginNames = uniq(Object.keys(space.plugins || {}));
-    const popularity = getPopularity(id, {
-      verified,
-      turbo,
-      networks,
-      strategies: strategyNames
-    });
 
     spacesMetadata[id] = {
       id,
       name: space.name,
-      verified,
-      flagged,
-      turbo,
-      hibernated,
-      popularity,
+      verified: space.verified,
+      flagged: space.flagged,
+      turbo: space.turbo,
+      hibernated: space.hibernated,
+      popularity: spacesMetadata[id]?.popularity || 0,
+      rank: spacesMetadata[id]?.rank || 0,
       private: space.private ?? false,
       categories: space.categories ?? [],
       networks,
       counts: {
-        activeProposals: spaceProposals[id]?.active || 0,
-        proposalsCount: spaceProposals[id]?.count || 0,
-        proposalsCount7d: spaceProposals[id]?.count_7d || 0,
-        followersCount: spaceFollowers[id]?.count || 0,
-        followersCount7d: spaceFollowers[id]?.count_7d || 0,
-        votesCount: spaceVotes[id]?.count || 0,
-        votesCount7d: spaceVotes[id]?.count_7d || 0
+        activeProposals: spacesMetadata[id]?.counts?.activeProposals || 0,
+        proposalsCount: space.proposal_count || 0,
+        proposalsCount7d: spacesMetadata[id]?.counts?.proposalsCount7d || 0,
+        followersCount: space.follower_count || 0,
+        followersCount7d: spacesMetadata[id]?.counts?.followersCount7d || 0,
+        votesCount: space.vote_count || 0,
+        votesCount7d: spacesMetadata[id]?.counts?.votesCount7d || 0
       },
       strategyNames,
       pluginNames
     };
   });
-
-  rankedSpaces = Object.values(spacesMetadata)
-    .filter((space: any) => !space.private && !space.flagged)
-    .sort((a: any, b: any) => b.popularity - a.popularity);
-
-  rankedSpaces.forEach((space: any, i: number) => {
-    spacesMetadata[space.id].rank = i + 1;
-  });
 }
 
 async function loadSpaces() {
   const query =
     'SELECT id, settings, flagged, verified, turbo, hibernated FROM spaces WHERE deleted = 0 ORDER BY id ASC';
   const s = await db.queryAsync(query);
   spaces = Object.fromEntries(
-    s.map(ensSpace => [
-      ensSpace.id,
+    s.map(space => [
+      space.id,
       {
-        ...JSON.parse(ensSpace.settings),
-        flagged: ensSpace.flagged === 1,
-        verified: ensSpace.verified === 1,
-        turbo: ensSpace.turbo === 1,
-        hibernated: ensSpace.hibernated === 1
+        ...JSON.parse(space.settings),
+        flagged: space.flagged === 1,
+        verified: space.verified === 1,
+        turbo: space.turbo === 1,
+        hibernated: space.hibernated === 1
       }
     ])
   );
@@ -131,68 +144,97 @@ async function loadSpaces() {
   mapSpaces();
 }
 
-async function getProposals() {
+async function getProposals(): Promise<
+  Record<string, { activeProposals: number; proposalsCount7d: number }>
+> {
   const ts = parseInt((Date.now() / 1e3).toFixed());
+  const results = {};
+
   const query = `
     SELECT
       space,
-      spaces.proposal_count AS count,
-      COUNT(IF(p.start < ? AND p.end > ? AND p.flagged = 0, 1, NULL)) AS active,
-      COUNT(IF(p.created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) AS count_7d,
-    FROM proposals p
-    JOIN spaces ON spaces.id = space
+      COUNT(id) AS proposalsCount7d
+    FROM proposals
+    WHERE created > (UNIX_TIMESTAMP() - 604800)
     GROUP BY space
   `;
-  return await db.queryAsync(query, [ts, ts]);
+
+  (await db.queryAsync(query)).forEach(({ space, proposalsCount7d }) => {
+    results[space] ||= {};
+    results[space].proposalsCount7d = proposalsCount7d;
+  });
+
+  const activeQuery = `
+    SELECT
+      space,
+      COUNT(id) AS activeProposals
+    FROM proposals
+    WHERE start < ? AND end > ? AND flagged = 0
+    GROUP BY space
+  `;
+
+  (await db.queryAsync(activeQuery, [ts, ts])).forEach(
+    ({ space, activeProposals }) => {
+      results[space] ||= {};
+      results[space].activeProposals = activeProposals;
+    }
+  );
+
+  return results;
 }
 
-async function getVotes() {
+async function getVotes(): Promise<Record<string, { votesCount7d: number }>> {
   const query = `
     SELECT
       space,
-      spaces.vote_count as count,
-      COUNT(IF(v.created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) AS count_7d
-    FROM votes v
-    JOIN spaces ON spaces.id = space
+      COUNT(id) AS votesCount7d
+    FROM votes
+    WHERE created > (UNIX_TIMESTAMP() - 604800)
     GROUP BY space
   `;
-  return await db.queryAsync(query);
+
+  return Object.fromEntries(
+    (await db.queryAsync(query)).map(({ space, votesCount7d }) => [
+      space,
+      { votesCount7d }
+    ])
+  );
 }
 
-async function getFollowers() {
+async function getFollowers(): Promise<
+  Record<string, { followersCount7d: number }>
+> {
   const query = `
     SELECT
       space,
-      spaces.follower_count AS count,
-      COUNT(IF(f.created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) AS count_7d
-    FROM follows f
-    JOIN spaces ON spaces.id = space
+      COUNT(id) AS followersCount7d
+    FROM follows
+    WHERE created > (UNIX_TIMESTAMP() - 604800)
     GROUP BY space
   `;
-  return await db.queryAsync(query);
+
+  return Object.fromEntries(
+    (await db.queryAsync(query)).map(({ space, followersCount7d }) => [
+      space,
+      { followersCount7d }
+    ])
+  );
 }
 
 async function loadSpacesMetrics() {
-  const followersMetrics = await getFollowers();
-  followersMetrics.forEach(followers => {
-    if (spaces[followers.space]) spaceFollowers[followers.space] = followers;
-  });
-  log.info('[spaces] Followers metrics loaded');
-  mapSpaces();
+  [getFollowers, getProposals, getVotes].forEach(async metricFn => {
+    const results = await metricFn();
 
-  const proposalsMetrics = await getProposals();
-  proposalsMetrics.forEach(proposals => {
-    if (spaces[proposals.space]) spaceProposals[proposals.space] = proposals;
-  });
-  log.info('[spaces] Proposals metrics loaded');
-  mapSpaces();
+    for (const [space, metrics] of Object.entries(results)) {
+      if (!spacesMetadata[space]) continue;
 
-  const votesMetrics = await getVotes();
-  votesMetrics.forEach(votes => {
-    if (spaces[votes.space]) spaceVotes[votes.space] = votes;
+      spacesMetadata[space].counts = {
+        ...spacesMetadata[space].counts,
+        ...metrics
+      };
+    }
+    log.info(`[spaces] ${metricFn.name.replace('get', '')} metrics loaded`);
   });
-  log.info('[spaces] Votes metrics loaded');
-  mapSpaces();
 }
 
 export async function getSpace(id: string) {
@@ -220,10 +262,11 @@ export default async function run() {
   try {
     await loadSpaces();
     await loadSpacesMetrics();
+    sortSpaces();
   } catch (e: any) {
     capture(e);
     log.error(`[spaces] failed to load spaces, ${JSON.stringify(e)}`);
   }
-  await snapshot.utils.sleep(120e3);
+  await snapshot.utils.sleep(RUN_INTERVAL);
   run();
 }