From e8d3b25a5e56f7a3174867befacade907118fbff Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 30 Sep 2024 16:17:11 +0200 Subject: [PATCH] add tests to the user sync worker and cronjob --- package.json | 1 + src/repositories/qfRoundRepository.test.ts | 56 ++++++++++++ src/repositories/qfRoundRepository.ts | 2 +- .../cronJobs/syncUsersModelScore.test.ts | 85 +++++++++++++++++++ src/services/cronJobs/syncUsersModelScore.ts | 36 +++----- ...yncWorker.ts => userMBDScoreSyncWorker.ts} | 4 +- 6 files changed, 156 insertions(+), 28 deletions(-) create mode 100644 src/services/cronJobs/syncUsersModelScore.test.ts rename src/workers/{usersMBDScoreSyncWorker.ts => userMBDScoreSyncWorker.ts} (85%) diff --git a/package.json b/package.json index 36d4c96ac..4555ea968 100644 --- a/package.json +++ b/package.json @@ -135,6 +135,7 @@ "test:qfRoundHistoryRepository": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/repositories/qfRoundHistoryRepository.test.ts", "test:qfRoundService": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/services/qfRoundService.test.ts", "test:project": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/entities/project.test.ts", + "test:syncUsersModelScore": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/services/cronJobs/syncUsersModelScore.test.ts", "test:notifyDonationsWithSegment": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/services/cronJobs/notifyDonationsWithSegment.test.ts", "test:checkProjectVerificationStatus": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/services/cronJobs/checkProjectVerificationStatus.test.ts", "test:statusReasonResolver": "NODE_ENV=test mocha ./test/pre-test-scripts.ts ./src/resolvers/statusReasonResolver.test.ts", diff --git a/src/repositories/qfRoundRepository.test.ts b/src/repositories/qfRoundRepository.test.ts index beacac09f..adab54b26 100644 --- a/src/repositories/qfRoundRepository.test.ts +++ b/src/repositories/qfRoundRepository.test.ts @@ -17,6 +17,7 @@ import { getProjectDonationsSqrtRootSum, getQfRoundTotalSqrtRootSumSquared, getQfRoundStats, + findUsersWithoutMBDScoreInActiveAround, } from './qfRoundRepository'; import { Project } from '../entities/project'; import { refreshProjectEstimatedMatchingView } from '../services/projectViewsService'; @@ -26,6 +27,11 @@ describe( 'getProjectDonationsSqrtRootSum test cases', getProjectDonationsSqrRootSumTests, ); + +describe( + 'findUsersWithoutMBDScoreInActiveAround test cases', + findUsersWithoutMBDScoreInActiveAroundTestCases, +); describe( 'getQfRoundTotalProjectsDonationsSum test cases', getQfRoundTotalProjectsDonationsSumTestCases, @@ -41,6 +47,56 @@ describe( describe('findQfRoundById test cases', findQfRoundByIdTestCases); describe('findQfRoundBySlug test cases', findQfRoundBySlugTestCases); +function findUsersWithoutMBDScoreInActiveAroundTestCases() { + it('should find users without score that donated in the round', async () => { + await QfRound.update({}, { isActive: false }); + const qfRound = QfRound.create({ + isActive: true, + name: 'test', + allocatedFund: 100, + minimumPassportScore: 8, + slug: new Date().getTime().toString(), + beginDate: new Date(), + endDate: moment().add(10, 'days').toDate(), + }); + await qfRound.save(); + const project = await saveProjectDirectlyToDb(createProjectData()); + project.qfRounds = [qfRound]; + await project.save(); + + const user = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + const user2 = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + await saveDonationDirectlyToDb( + { + ...createDonationData(), + segmentNotified: false, + qfRoundId: qfRound.id, + status: 'verified', + }, + user.id, + project.id, + ); + + await saveDonationDirectlyToDb( + { + ...createDonationData(), + segmentNotified: false, + qfRoundId: qfRound.id, + status: 'verified', + }, + user2.id, + project.id, + ); + + const userIds = await findUsersWithoutMBDScoreInActiveAround(); + assert.equal(userIds.length, 2); + assert.isTrue(userIds.includes(user.id) && userIds.includes(user2.id)); + + qfRound.isActive = false; + await qfRound.save(); + }); +} + function getProjectDonationsSqrRootSumTests() { let qfRound: QfRound; let project: Project; diff --git a/src/repositories/qfRoundRepository.ts b/src/repositories/qfRoundRepository.ts index 162da7497..91c42749b 100644 --- a/src/repositories/qfRoundRepository.ts +++ b/src/repositories/qfRoundRepository.ts @@ -198,7 +198,7 @@ export const findUsersWithoutMBDScoreInActiveAround = async (): Promise< [activeQfRoundId], ); - return usersMissingMDBScore; + return usersMissingMDBScore.map(user => user.userId); }; export const findQfRoundById = async (id: number): Promise => { diff --git a/src/services/cronJobs/syncUsersModelScore.test.ts b/src/services/cronJobs/syncUsersModelScore.test.ts new file mode 100644 index 000000000..321109845 --- /dev/null +++ b/src/services/cronJobs/syncUsersModelScore.test.ts @@ -0,0 +1,85 @@ +import { assert } from 'chai'; +import moment from 'moment'; +import { + createDonationData, + createProjectData, + generateRandomEtheriumAddress, + saveDonationDirectlyToDb, + saveProjectDirectlyToDb, + saveUserDirectlyToDb, +} from '../../../test/testUtils'; +import { QfRound } from '../../entities/qfRound'; +import { updateUsersWithoutMBDScoreInRound } from './syncUsersModelScore'; +import { UserQfRoundModelScore } from '../../entities/userQfRoundModelScore'; + +describe( + 'updateUsersWithoutMBDScoreInRound() test cases', + updateUsersWithoutMBDScoreInRoundTestCases, +); + +function updateUsersWithoutMBDScoreInRoundTestCases() { + // for tests it return 1, useful to test cronjob logic and worker + it('should save the score for users that donated in the round', async () => { + await QfRound.update({}, { isActive: false }); + const qfRound = QfRound.create({ + isActive: true, + name: 'test', + allocatedFund: 100, + minimumPassportScore: 8, + slug: new Date().getTime().toString(), + beginDate: new Date(), + endDate: moment().add(10, 'days').toDate(), + }); + await qfRound.save(); + const project = await saveProjectDirectlyToDb(createProjectData()); + project.qfRounds = [qfRound]; + await project.save(); + + const user = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + const user2 = await saveUserDirectlyToDb(generateRandomEtheriumAddress()); + await saveDonationDirectlyToDb( + { + ...createDonationData(), + segmentNotified: false, + qfRoundId: qfRound.id, + status: 'verified', + }, + user.id, + project.id, + ); + + await saveDonationDirectlyToDb( + { + ...createDonationData(), + segmentNotified: false, + qfRoundId: qfRound.id, + status: 'verified', + }, + user2.id, + project.id, + ); + + await updateUsersWithoutMBDScoreInRound(); + + const user1ModelScore = await UserQfRoundModelScore.createQueryBuilder( + 'score', + ) + .where('score."userId" = :userId', { userId: user.id }) + .andWhere('score."qfRoundId" = :qfRoundId', { qfRoundId: qfRound.id }) + .getOne(); + + const user2ModelScore = await UserQfRoundModelScore.createQueryBuilder( + 'score', + ) + .where('score."userId" = :userId', { userId: user2.id }) + .andWhere('score."qfRoundId" = :qfRoundId', { qfRoundId: qfRound.id }) + .getOne(); + + // base values for mocks + assert.equal(user1ModelScore?.score, 1); + assert.equal(user2ModelScore?.score, 1); + + qfRound.isActive = false; + await qfRound.save(); + }); +} diff --git a/src/services/cronJobs/syncUsersModelScore.ts b/src/services/cronJobs/syncUsersModelScore.ts index f37fecf62..944363e6a 100644 --- a/src/services/cronJobs/syncUsersModelScore.ts +++ b/src/services/cronJobs/syncUsersModelScore.ts @@ -1,32 +1,22 @@ import { schedule } from 'node-cron'; -import { ModuleThread, Pool, spawn, Worker } from 'threads'; +import { spawn, Worker, Thread } from 'threads'; import config from '../../config'; import { logger } from '../../utils/logger'; import { findActiveQfRound, findUsersWithoutMBDScoreInActiveAround, } from '../../repositories/qfRoundRepository'; -import { UserMDBScoreSyncWorker } from '../../workers/usersMBDScoreSyncWorker'; import { findUserById } from '../../repositories/userRepository'; import { UserQfRoundModelScore } from '../../entities/userQfRoundModelScore'; const cronJobTime = (config.get('MAKE_UNREVIEWED_PROJECT_LISTED_CRONJOB_EXPRESSION') as string) || - '0 0 * * *'; + '0 0 * * * *'; const qfRoundUsersMissedMBDScore = Number( process.env.QF_ROUND_USERS_MISSED_SCORE || 0, ); -const workerOptions = { - concurrency: Number( - process.env.USER_SCORE_SYNC_THREADS_POOL_CONCURRENCY || 1, - ), - name: - process.env.USER_SCORE_SYNC_THREADS_POOL_NAME || 'ProjectFiltersThreadPool', - size: Number(process.env.USER_SCORE_SYNC_THREADS_POOL_SIZE || 4), -}; - export const runCheckPendingUserModelScoreCronjob = () => { logger.debug( 'runCheckPendingUserModelScoreCronjob() has been called, cronJobTime', @@ -38,13 +28,9 @@ export const runCheckPendingUserModelScoreCronjob = () => { }; export const updateUsersWithoutMBDScoreInRound = async () => { - const usersMDBScoreSyncThreadPool: Pool< - ModuleThread - > = Pool( - () => spawn(new Worker('../workers/usersMBDScoreSyncWoker')), - workerOptions, + const worker = await spawn( + new Worker('../../workers/userMBDScoreSyncWorker'), ); - const userIds = await findUsersWithoutMBDScoreInActiveAround(); const activeQfRoundId = (await findActiveQfRound())?.id || qfRoundUsersMissedMBDScore; @@ -54,12 +40,12 @@ export const updateUsersWithoutMBDScoreInRound = async () => { for (const userId of userIds) { try { - const userWallet = await findUserById(userId); - const userScore = await usersMDBScoreSyncThreadPool.queue(worker => - worker.syncUserScore({ - userWallet, - }), - ); + const user = await findUserById(userId); + if (!user) continue; + + const userScore = await worker.syncUserScore({ + userWallet: user?.walletAddress, + }); if (userScore) { const userScoreInRound = UserQfRoundModelScore.create({ userId, @@ -73,5 +59,5 @@ export const updateUsersWithoutMBDScoreInRound = async () => { logger.info(`User with Id ${userId} did not sync MBD score this batch`); } } - await usersMDBScoreSyncThreadPool.terminate(); + await Thread.terminate(worker); }; diff --git a/src/workers/usersMBDScoreSyncWorker.ts b/src/workers/userMBDScoreSyncWorker.ts similarity index 85% rename from src/workers/usersMBDScoreSyncWorker.ts rename to src/workers/userMBDScoreSyncWorker.ts index fdbd37420..0bcc65d4c 100644 --- a/src/workers/usersMBDScoreSyncWorker.ts +++ b/src/workers/userMBDScoreSyncWorker.ts @@ -5,10 +5,10 @@ import { getGitcoinAdapter } from '../adapters/adaptersFactory'; type UsersMBDScoreSyncWorkerFunctions = 'syncUserScore'; -export type UserMDBScoreSyncWorker = +export type UserMBDScoreSyncWorker = WorkerModule; -const worker: UserMDBScoreSyncWorker = { +const worker: UserMBDScoreSyncWorker = { async syncUserScore(args: { userWallet: string }) { return await getGitcoinAdapter().getUserAnalysisScore(args.userWallet); },