Skip to content

Commit

Permalink
fix: optimize spaces helpers (#887)
Browse files Browse the repository at this point in the history 
* fix: use new counter columns from table spaces

* fix ambiguousfields

* fix: improve queries performance

* fix: improve types

* fix: fix promises not being waited

* fix: fix missing counter from spaces table

* fix: use faster single query instead of batch

* chore: remove unused variable

---------

Co-authored-by: ChaituVR <[email protected]>
  • Loading branch information
@wa0x6e @ChaituVR
wa0x6e and ChaituVR authored Jul 22, 2024
1 parent c2cdbc5 commit 6269b94
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 110 deletions.
2 changes: 1 addition & 1 deletion src/graphql/operations/ranking.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export default async function (_parent, args, context, info) {
let searchCategory = category?.toLowerCase() || '';
if (searchCategory === 'all') searchCategory = '';

let filteredSpaces = rankedSpaces.filter((space: any) => {
let filteredSpaces = rankedSpaces.filter(space => {
const filteredBySearch =
space.id.includes(searchStr) ||
space.name?.toLowerCase().includes(searchStr);
Expand Down
294 changes: 185 additions & 109 deletions src/helpers/spaces.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,61 +5,90 @@ import db from './mysql';
import log from './log';
import { capture } from '@snapshot-labs/snapshot-sentry';

export let spaces = {};
export const spacesMetadata = {};
export let rankedSpaces: any = [];
const spaceProposals = {};
const spaceVotes = {};
const spaceFollowers = {};

const testnets = Object.values(networks)
.filter((network: any) => network.testnet)
.map((network: any) => network.key);
const testStrategies = [
const RUN_INTERVAL = 120e3;
const TEST_STRATEGIES = [
'ticket',
'api',
'api-v2',
'api-post',
'api-v2-override'
];
const TESTNET_NETWORKS = (
Object.values(networks) as { testnet?: boolean; key: string }[]
)
.filter(network => network.testnet)
.map(network => network.key);

function getPopularity(
id: string,
params: {
verified: boolean;
turbo: boolean;
networks: string[];
strategies: any[];
}
): number {
export let spaces = {};
export let rankedSpaces: Metadata[] = [];
export const spacesMetadata: Record<string, Metadata> = {};

type Metadata = {
id: string;
name: string;
verified: boolean;
flagged: boolean;
turbo: boolean;
hibernated: boolean;
popularity: number;
rank: number | null;
private: boolean;
categories: string[];
networks: string[];
counts: {
activeProposals: number;
proposalsCount: number;
proposalsCount7d: number;
followersCount: number;
followersCount7d: number;
votesCount: number;
votesCount7d: number;
};
strategyNames: string[];
pluginNames: string[];
};

function getPopularity(space: Metadata): number {
let popularity =
(spaceVotes[id]?.count || 0) / 100 +
(spaceVotes[id]?.count_7d || 0) +
(spaceProposals[id]?.count || 0) / 100 +
(spaceProposals[id]?.count_7d || 0) +
(spaceFollowers[id]?.count || 0) / 50 +
(spaceFollowers[id]?.count_7d || 0);

if (params.networks.some(network => testnets.includes(network)))
popularity = 1;
if (params.strategies.some(strategy => testStrategies.includes(strategy)))
space.counts.votesCount / 100 +
space.counts.votesCount7d +
space.counts.proposalsCount / 100 +
space.counts.proposalsCount7d +
space.counts.followersCount / 50 +
space.counts.followersCount7d;

if (
space.networks.some(network => TESTNET_NETWORKS.includes(network)) ||
space.strategyNames.some(strategy => TEST_STRATEGIES.includes(strategy))
)
popularity = 1;

if (params.verified) popularity *= 100000000;
if (params.turbo) {
if (space.verified) popularity *= 100000000;
if (space.turbo) {
popularity += 1;
popularity *= 100000000;
}

return popularity;
}

function sortSpaces() {
Object.entries(spacesMetadata).forEach(([id, space]) => {
spacesMetadata[id].popularity = getPopularity(space);
});

rankedSpaces = Object.values(spacesMetadata)
.filter(space => !space.private && !space.flagged && space.popularity > 0)
.sort((a, b) => b.popularity - a.popularity)
.map((space, i) => {
spacesMetadata[space.id].rank = i + 1;
space.rank = i + 1;
return space;
});
}

function mapSpaces() {
Object.entries(spaces).forEach(([id, space]: any) => {
const verified = space.verified || false;
const flagged = space.flagged || false;
const turbo = space.turbo || false;
const hibernated = space.hibernated || false;
const networks = uniq(
(space.strategies || [])
.map(strategy => strategy?.network || space.network)
Expand All @@ -69,123 +98,167 @@ function mapSpaces() {
(space.strategies || []).map(strategy => strategy.name)
);
const pluginNames = uniq(Object.keys(space.plugins || {}));
const popularity = getPopularity(id, {
verified,
turbo,
networks,
strategies: strategyNames
});

spacesMetadata[id] = {
id,
name: space.name,
verified,
flagged,
turbo,
hibernated,
popularity,
verified: space.verified,
flagged: space.flagged,
turbo: space.turbo,
hibernated: space.hibernated,
popularity: spacesMetadata[id]?.popularity || 0,
rank: spacesMetadata[id]?.rank || null,
private: space.private ?? false,
categories: space.categories ?? [],
networks,
counts: {
activeProposals: spaceProposals[id]?.active || 0,
proposalsCount: spaceProposals[id]?.count || 0,
proposalsCount7d: spaceProposals[id]?.count_7d || 0,
followersCount: spaceFollowers[id]?.count || 0,
followersCount7d: spaceFollowers[id]?.count_7d || 0,
votesCount: spaceVotes[id]?.count || 0,
votesCount7d: spaceVotes[id]?.count_7d || 0
activeProposals: spacesMetadata[id]?.counts?.activeProposals || 0,
proposalsCount: space.proposal_count || 0,
proposalsCount7d: spacesMetadata[id]?.counts?.proposalsCount7d || 0,
followersCount: space.follower_count || 0,
followersCount7d: spacesMetadata[id]?.counts?.followersCount7d || 0,
votesCount: space.vote_count || 0,
votesCount7d: spacesMetadata[id]?.counts?.votesCount7d || 0
},
strategyNames,
pluginNames
};
});

rankedSpaces = Object.values(spacesMetadata)
.filter((space: any) => !space.private && !space.flagged)
.sort((a: any, b: any) => b.popularity - a.popularity);

rankedSpaces.forEach((space: any, i: number) => {
spacesMetadata[space.id].rank = i + 1;
});
}

async function loadSpaces() {
const query =
'SELECT id, settings, flagged, verified, turbo, hibernated FROM spaces WHERE deleted = 0 ORDER BY id ASC';
const s = await db.queryAsync(query);
const startTime = +Date.now();

const query = `
SELECT id, settings, flagged, verified, turbo, hibernated, follower_count, proposal_count, vote_count
FROM spaces
WHERE deleted = 0
ORDER BY id ASC
`;
const results = await db.queryAsync(query);

spaces = Object.fromEntries(
s.map(ensSpace => [
ensSpace.id,
results.map(space => [
space.id,
{
...JSON.parse(ensSpace.settings),
flagged: ensSpace.flagged === 1,
verified: ensSpace.verified === 1,
turbo: ensSpace.turbo === 1,
hibernated: ensSpace.hibernated === 1
...JSON.parse(space.settings),
flagged: space.flagged === 1,
verified: space.verified === 1,
turbo: space.turbo === 1,
hibernated: space.hibernated === 1,
follower_count: space.follower_count,
vote_count: space.vote_count,
proposal_count: space.proposal_count
}
])
);
const totalSpaces = Object.keys(spaces).length;
log.info(`[spaces] total spaces ${totalSpaces}`);

log.info(
`[spaces] total spaces ${Object.keys(spaces).length}, in (${(
(+Date.now() - startTime) /
1000
).toFixed()}s)`
);
mapSpaces();
}

async function getProposals() {
async function getProposals(): Promise<
Record<string, { activeProposals: number; proposalsCount7d: number }>
> {
const ts = parseInt((Date.now() / 1e3).toFixed());
const results = {};

const query = `
SELECT space, COUNT(id) AS count,
COUNT(IF(start < ? AND end > ? AND flagged = 0, 1, NULL)) AS active,
count(IF(created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) as count_7d
FROM proposals GROUP BY space
SELECT
space,
COUNT(id) AS proposalsCount7d
FROM proposals
WHERE created > (UNIX_TIMESTAMP() - 604800)
GROUP BY space
`;
return await db.queryAsync(query, [ts, ts]);

(await db.queryAsync(query)).forEach(({ space, proposalsCount7d }) => {
results[space] ||= {};
results[space].proposalsCount7d = proposalsCount7d;
});

const activeQuery = `
SELECT
space,
COUNT(id) AS activeProposals
FROM proposals
WHERE start < ? AND end > ? AND flagged = 0
GROUP BY space
`;

(await db.queryAsync(activeQuery, [ts, ts])).forEach(
({ space, activeProposals }) => {
results[space] ||= {};
results[space].activeProposals = activeProposals;
}
);

return results;
}

async function getVotes() {
async function getVotes(): Promise<Record<string, { votesCount7d: number }>> {
const query = `
SELECT space, COUNT(id) as count,
count(IF(created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) as count_7d
FROM votes GROUP BY space
SELECT
space,
COUNT(id) AS votesCount7d
FROM votes
WHERE created > (UNIX_TIMESTAMP() - 604800)
GROUP BY space
`;
return await db.queryAsync(query);

return Object.fromEntries(
(await db.queryAsync(query)).map(({ space, votesCount7d }) => [
space,
{ votesCount7d }
])
);
}

async function getFollowers() {
async function getFollowers(): Promise<
Record<string, { followersCount7d: number }>
> {
const query = `
SELECT space, COUNT(id) as count,
count(IF(created > (UNIX_TIMESTAMP() - 604800), 1, NULL)) as count_7d
FROM follows GROUP BY space
SELECT
space,
COUNT(id) AS followersCount7d
FROM follows
WHERE created > (UNIX_TIMESTAMP() - 604800)
GROUP BY space
`;
return await db.queryAsync(query);

return Object.fromEntries(
(await db.queryAsync(query)).map(({ space, followersCount7d }) => [
space,
{ followersCount7d }
])
);
}

async function loadSpacesMetrics() {
const followersMetrics = await getFollowers();
followersMetrics.forEach(followers => {
if (spaces[followers.space]) spaceFollowers[followers.space] = followers;
});
log.info('[spaces] Followers metrics loaded');
mapSpaces();
const metricsFn = [getFollowers, getProposals, getVotes];
const results = await Promise.all(metricsFn.map(fn => fn()));

const proposalsMetrics = await getProposals();
proposalsMetrics.forEach(proposals => {
if (spaces[proposals.space]) spaceProposals[proposals.space] = proposals;
});
log.info('[spaces] Proposals metrics loaded');
mapSpaces();
metricsFn.forEach((metricFn, i) => {
for (const [space, metrics] of Object.entries(results[i])) {
if (!spacesMetadata[space]) continue;

const votesMetrics = await getVotes();
votesMetrics.forEach(votes => {
if (spaces[votes.space]) spaceVotes[votes.space] = votes;
spacesMetadata[space].counts = {
...spacesMetadata[space].counts,
...metrics
};
}
log.info(`[spaces] ${metricFn.name.replace('get', '')} metrics loaded`);
});
log.info('[spaces] Votes metrics loaded');
mapSpaces();
}

export async function getSpace(id: string) {
const query = `
SELECT settings, flagged, verified, turbo, hibernated, deleted
SELECT settings, flagged, verified, turbo, hibernated, deleted, follower_count, proposal_count, vote_count
FROM spaces
WHERE id = ?
LIMIT 1`;
Expand All @@ -206,12 +279,15 @@ export async function getSpace(id: string) {

export default async function run() {
try {
log.info('[spaces] Start spaces refresh');
await loadSpaces();
await loadSpacesMetrics();
sortSpaces();
log.info('[spaces] End spaces refresh');
} catch (e: any) {
capture(e);
log.error(`[spaces] failed to load spaces, ${JSON.stringify(e)}`);
}
await snapshot.utils.sleep(120e3);
await snapshot.utils.sleep(RUN_INTERVAL);
run();
}

0 comments on commit 6269b94

Please sign in to comment.