From 5d7ec81283b54862933174ba27204fc78a7d3ebe Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 19 Mar 2024 16:36:22 -0400 Subject: [PATCH 01/75] wip --- src/lib/cache.ts | 2 +- src/services/dashboards/resource.js | 339 +++++++++++++++++++---- src/services/dashboards/resource.test.js | 254 +++++++++++++++-- src/services/users.js | 1 + 4 files changed, 515 insertions(+), 81 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index d62450df90..e724457d7f 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (!ignoreCache) { + if (false) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index f3827394a0..b88171d52e 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -1,6 +1,7 @@ /* eslint-disable max-len */ -import { Sequelize, Op } from 'sequelize'; +import { Sequelize, Op, QueryTypes } from 'sequelize'; import { REPORT_STATUSES } from '@ttahub/common'; +import { v4 as uuidv4 } from 'uuid'; import { ActivityReport, ActivityReportGoal, @@ -330,6 +331,223 @@ const switchToTopicCentric = (input) => { }); }; +/* + Create a flat table to calculate the resource data. Use temp tables to ONLY join to the rows we need. + If over time the amount of data increases and slows again we can cache the flat table a set frequency. +*/ +export async function resourceFlatData(scopes) { + console.time('flatTime'); + // Date to retrieve report data from. + const reportCreatedAtDate = '2022-12-01'; + + // Get all ActivityReport ID's using SCOPES. + // We don't want to write custom filters. + console.time('scopesTime'); + const reportIds = await ActivityReport.findAll({ + attributes: [ + 'id', + ], + where: { + [Op.and]: [ + scopes.activityReport, + { + calculatedStatus: REPORT_STATUSES.APPROVED, + startDate: { [Op.ne]: null }, + createdAt: { [Op.gt]: reportCreatedAtDate }, + }, + ], + }, + raw: true, + }); + console.timeEnd('scopesTime'); + console.log('\n\n\n------ AR IDS from Scopes: ', reportIds); + + // Write raw sql to generate the flat resource data for the above reportIds. + const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; + const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; + const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; + const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; + const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; + const createdAroTopicsRolledUpTempTableName = `Z_temp_resource_aro_rolled_up_topics__${uuidv4().replaceAll('-', '_')}`; + const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. + + // Create raw sql to get flat table. + const flatResourceSql = ` + -- 1.) Create AR temp table. + SELECT + id, + "startDate", + "numberOfParticipants", + to_char("startDate", 'Mon-YY') AS "rollUpDate", + "regionId", + "calculatedStatus" + INTO TEMP ${createdArTempTableName} + FROM "ActivityReports" ar + WHERE ar."id" IN (${reportIds.map((r) => r.id).join(',')}); + + -- 2.) Create ARO Resources temp table. + SELECT + ar.id AS "activityReportId", + aror."resourceId" + INTO TEMP ${createdAroResourcesTempTableName} + FROM ${createdArTempTableName} ar + JOIN "ActivityReportObjectives" aro + ON ar."id" = aro."activityReportId" + JOIN "ActivityReportObjectiveResources" aror + ON aro.id = aror."activityReportObjectiveId" + WHERE aror."sourceFields" = '{resource}' + GROUP BY ar.id, aror."resourceId"; + + -- 3.) Create Resources temp table (only what we need). + SELECT + id, + domain, + url, + title + INTO TEMP ${createdResourcesTempTableName} + FROM "Resources" + WHERE id IN ( + SELECT DISTINCT "resourceId" + FROM ${createdAroResourcesTempTableName} + ); + + -- 4.) Create ARO Topics temp table. + SELECT + ar.id AS "activityReportId", + arot."topicId" + INTO TEMP ${createdAroTopicsTempTableName} + FROM ${createdArTempTableName} ar + JOIN "ActivityReportObjectives" aro + ON ar."id" = aro."activityReportId" + JOIN "ActivityReportObjectiveTopics" arot + ON aro.id = arot."activityReportObjectiveId" + GROUP BY ar.id, arot."topicId"; + + -- 5.) Create Topics temp table (only what we need). + SELECT + id, + name + INTO TEMP ${createdTopicsTempTableName} + FROM "Topics" + WHERE id IN ( + SELECT DISTINCT "topicId" + FROM ${createdAroTopicsTempTableName} + ); + + -- 6.) Create Rolled Up Topics temp table (maybe delete this later...). + SELECT + arot."activityReportId", + ARRAY_AGG(DISTINCT arott.name) AS topics + INTO TEMP ${createdAroTopicsRolledUpTempTableName} + FROM ${createdAroTopicsTempTableName} arot + JOIN ${createdTopicsTempTableName} arott + ON arot."topicId" = arott.id + GROUP BY arot."activityReportId"; + + -- 7.) Create Flat Resource temp table. + SELECT + ar.id, + ar."startDate", + ar."rollUpDate", + artog.topics, + arorr.domain, + arorr.title, + arorr.url + INTO TEMP ${createdFlatResourceTempTableName} + FROM ${createdArTempTableName} ar + JOIN ${createdAroResourcesTempTableName} aror + ON ar.id = aror."activityReportId" + JOIN ${createdResourcesTempTableName} arorr + ON aror."resourceId" = arorr.id + LEFT JOIN ${createdAroTopicsRolledUpTempTableName} artog + ON ar.id = artog."activityReportId"; + `; + console.log('\n\n\n------- SQL BEFORE: ', flatResourceSql); + console.time('sqlTime'); + // await sequelize.query('SELECT * FROM projects', { raw: true }); + const transaction = await sequelize.transaction(); + + // Create base tables. + await sequelize.query( + flatResourceSql, + { + // raw: true, + // nest: false, + type: QueryTypes.SELECT, + // mapToModel: false, + transaction, + }, + ); + + // Select the Flat table. + /* + let flatTable = sequelize.query( + `SELECT * FROM ${createdFlatResourceTempTableName};`, + { + type: QueryTypes.SELECT, + transaction, + }, + ); + */ + + // Get resource use result. + let resourceUseResult = sequelize.query( + ` + SELECT + url, + "rollUpDate", + count(id) AS "resourceCount" + FROM ${createdFlatResourceTempTableName} tf + GROUP BY url, "rollUpDate" + ORDER BY "url", tf."rollUpDate" ASC + LIMIT 10; + `, + { + type: QueryTypes.SELECT, + transaction, + }, + ); + + // Get final topic use result. + let topicUseResult = sequelize.query(` + SELECT + t.name, + f."rollUpDate", + count(f.id) AS "resourceCount" + FROM ${createdTopicsTempTableName} t + JOIN ${createdAroTopicsTempTableName} arot + ON t.id = arot."topicId" + JOIN ${createdFlatResourceTempTableName} f + ON arot."activityReportId" = f.id + GROUP BY t.name, f."rollUpDate" + ORDER BY t.name, f."rollUpDate" ASC; + `, { + type: QueryTypes.SELECT, + transaction, + }); + + // [flatTable, resourceUseResult, topicUseResult] = await Promise.all([flatTable, resourceUseResult, topicUseResult]); + [resourceUseResult, topicUseResult] = await Promise.all([resourceUseResult, topicUseResult]); + + // Commit is required to run the query. + transaction.commit(); + // console.log('\n\n\n------- SQL FLAT: ', flatTable); + console.log('\n\n\n------- SQL RESOURCE USE: ', resourceUseResult); + console.log('\n\n\n------- SQL TOPIC USE: ', topicUseResult); + /* + const tables = await sequelize.query( + `SELECT "name" FROM "Users" LIMIT 1; + SELECT "id" FROM "ActivityReports" LIMIT 2;`, + { + type: QueryTypes.SELECT, + }, + ); + */ + console.timeEnd('sqlTime'); + console.timeEnd('flatTime'); + return { resourceUseResult, topicUseResult }; +} + // collect all resource data from the db filtered via the scopes export async function resourceData(scopes, skipResources = false, skipTopics = false) { // Date to retrieve report data from. @@ -343,71 +561,74 @@ export async function resourceData(scopes, skipResources = false, skipTopics = f viaObjectives: null, viaGoals: null, }; - [ - dbData.allReports, - // dbData.viaReport, - // dbData.viaSpecialistNextSteps, - // dbData.viaRecipientNextSteps, - dbData.viaObjectives, - dbData.viaGoals, - ] = await Promise.all([ - await ActivityReport.findAll({ - attributes: [ - 'id', - 'numberOfParticipants', - 'topics', - 'startDate', - [sequelize.fn( - 'jsonb_agg', + console.time('reportsTime2'); + dbData.allReports = await ActivityReport.findAll({ + attributes: [ + 'id', + 'numberOfParticipants', + 'topics', + 'startDate', + [sequelize.fn( + 'jsonb_agg', + sequelize.fn( + 'DISTINCT', sequelize.fn( - 'DISTINCT', - sequelize.fn( - 'jsonb_build_object', - sequelize.literal('\'grantId\''), - sequelize.literal('"activityRecipients->grant"."id"'), - sequelize.literal('\'recipientId\''), - sequelize.literal('"activityRecipients->grant"."recipientId"'), - sequelize.literal('\'otherEntityId\''), - sequelize.literal('"activityRecipients"."otherEntityId"'), - ), + 'jsonb_build_object', + sequelize.literal('\'grantId\''), + sequelize.literal('"activityRecipients->grant"."id"'), + sequelize.literal('\'recipientId\''), + sequelize.literal('"activityRecipients->grant"."recipientId"'), + sequelize.literal('\'otherEntityId\''), + sequelize.literal('"activityRecipients"."otherEntityId"'), ), ), - 'recipients'], - ], - group: [ - '"ActivityReport"."id"', - '"ActivityReport"."numberOfParticipants"', - '"ActivityReport"."topics"', - '"ActivityReport"."startDate"', + ), + 'recipients'], + ], + group: [ + '"ActivityReport"."id"', + '"ActivityReport"."numberOfParticipants"', + '"ActivityReport"."topics"', + '"ActivityReport"."startDate"', + ], + where: { + [Op.and]: [ + scopes.activityReport, + { + calculatedStatus: REPORT_STATUSES.APPROVED, + startDate: { [Op.ne]: null }, + createdAt: { [Op.gt]: reportCreatedAtDate }, + }, ], - where: { - [Op.and]: [ - scopes.activityReport, + }, + include: [ + { + model: ActivityRecipient.scope(), + as: 'activityRecipients', + attributes: [], + required: true, + include: [ { - calculatedStatus: REPORT_STATUSES.APPROVED, - startDate: { [Op.ne]: null }, - createdAt: { [Op.gt]: reportCreatedAtDate }, + model: Grant.scope(), + as: 'grant', + attributes: [], + required: false, }, ], }, - include: [ - { - model: ActivityRecipient.scope(), - as: 'activityRecipients', - attributes: [], - required: true, - include: [ - { - model: Grant.scope(), - as: 'grant', - attributes: [], - required: false, - }, - ], - }, - ], - raw: true, - }), + ], + raw: true, + }); + + console.timeEnd('reportsTime2'); + [ + // dbData.allReports, + // dbData.viaReport, + // dbData.viaSpecialistNextSteps, + // dbData.viaRecipientNextSteps, + dbData.viaObjectives, + dbData.viaGoals, + ] = await Promise.all([ /* await ActivityReport.findAll({ attributes: [ @@ -1586,6 +1807,7 @@ export async function resourceTopicUse(scopes) { } export async function resourceDashboardPhase1(scopes) { + console.log('\n\n\n------Phase1'); const data = await resourceData(scopes); return { overview: generateResourcesDashboardOverview(data), @@ -1596,6 +1818,7 @@ export async function resourceDashboardPhase1(scopes) { } export async function resourceDashboard(scopes) { + console.log('\n\n\n------Old'); const data = await resourceData(scopes); return { overview: generateResourcesDashboardOverview(data), diff --git a/src/services/dashboards/resource.test.js b/src/services/dashboards/resource.test.js index 9490c337a2..6e59a552d1 100644 --- a/src/services/dashboards/resource.test.js +++ b/src/services/dashboards/resource.test.js @@ -21,6 +21,7 @@ import { resourceUse, resourceDashboard, resourceTopicUse, + resourceFlatData, } from './resource'; import { RESOURCE_DOMAIN } from '../../constants'; import { processActivityReportObjectiveForResourcesById } from '../resource'; @@ -87,7 +88,7 @@ const reportObject = { targetPopulations: ['pop'], reason: ['reason'], participants: ['participants'], - topics: ['Coaching'], + // topics: ['Coaching'], ttaType: ['technical-assistance'], version: 2, }; @@ -98,7 +99,7 @@ const regionOneReportA = { duration: 1, startDate: '2021-01-02T12:00:00Z', endDate: '2021-01-31T12:00:00Z', - topics: ['Coaching', 'ERSEA'], + // topics: ['Coaching', 'ERSEA'], }; const regionOneReportB = { @@ -107,7 +108,7 @@ const regionOneReportB = { duration: 2, startDate: '2021-01-15T12:00:00Z', endDate: '2021-02-15T12:00:00Z', - topics: ['Oral Health'], + // topics: ['Oral Health'], }; const regionOneReportC = { @@ -116,7 +117,7 @@ const regionOneReportC = { duration: 3, startDate: '2021-01-20T12:00:00Z', endDate: '2021-02-28T12:00:00Z', - topics: ['Nutrition'], + // topics: ['Nutrition'], }; const regionOneReportD = { @@ -125,7 +126,7 @@ const regionOneReportD = { duration: 3, startDate: '2021-01-22T12:00:00Z', endDate: '2021-01-31T12:00:00Z', - topics: ['Facilities', 'Fiscal / Budget', 'ERSEA'], + // topics: ['Facilities', 'Fiscal / Budget', 'ERSEA'], }; const regionOneDraftReport = { @@ -136,15 +137,19 @@ const regionOneDraftReport = { endDate: '2021-01-31T12:00:00Z', submissionStatus: REPORT_STATUSES.DRAFT, calculatedStatus: REPORT_STATUSES.DRAFT, - topics: ['Equity', 'ERSEA'], + // topics: ['Equity', 'ERSEA'], }; let grant; let goal; let objective; -let activityReportObjectiveOne; +let goalTwo; +let objectiveTwo; +let activityReportOneObjectiveOne; +let activityReportOneObjectiveTwo; let activityReportObjectiveTwo; let activityReportObjectiveThree; +let arIds; describe('Resources dashboard', () => { beforeAll(async () => { @@ -156,6 +161,7 @@ describe('Resources dashboard', () => { individualHooks: true, }); [goal] = await Goal.findOrCreate({ where: mockGoal, validate: true, individualHooks: true }); + [goalTwo] = await Goal.findOrCreate({ where: { ...mockGoal, name: 'Goal 2' }, validate: true, individualHooks: true }); [objective] = await Objective.findOrCreate({ where: { title: 'Objective 1', @@ -164,17 +170,76 @@ describe('Resources dashboard', () => { }, }); + [objectiveTwo] = await Objective.findOrCreate({ + where: { + title: 'Objective 2', + goalId: goalTwo.dataValues.id, + status: 'In Progress', + }, + }); + + // Get topic ID's. + const { topicId: classOrgTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'CLASS: Classroom Organization' }, + raw: true, + }); + + const { topicId: erseaTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'ERSEA' }, + raw: true, + }); + + const { topicId: coachingTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Coaching' }, + raw: true, + }); + + const { topicId: facilitiesTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Facilities' }, + raw: true, + }); + + const { topicId: fiscalBudgetTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Fiscal / Budget' }, + raw: true, + }); + + const { topicId: nutritionTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Nutrition' }, + raw: true, + }); + + const { topicId: oralHealthTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Oral Health' }, + raw: true, + }); + + const { topicId: equityTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Equity' }, + raw: true, + }); + // Report 1 (Mixed Resources). const reportOne = await ActivityReport.create({ ...regionOneReportA, }, { individualHooks: true, }); + console.log("\n\n\n------ Report 1"); await ActivityRecipient.findOrCreate({ where: { activityReportId: reportOne.id, grantId: mockGrant.id }, }); - [activityReportObjectiveOne] = await ActivityReportObjective.findOrCreate({ + // Report 1 - Activity Report Objective 1 + [activityReportOneObjectiveOne] = await ActivityReportObjective.findOrCreate({ where: { activityReportId: reportOne.id, status: 'Complete', @@ -182,23 +247,52 @@ describe('Resources dashboard', () => { }, }); - const { topicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'CLASS: Classroom Organization' }, - raw: true, + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: classOrgTopicId, + }, }); await ActivityReportObjectiveTopic.findOrCreate({ where: { - activityReportObjectiveId: activityReportObjectiveOne.id, - topicId, + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: erseaTopicId, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: coachingTopicId, }, }); // Report 1 ECLKC Resource 1. // Report 1 Non-ECLKC Resource 1. await processActivityReportObjectiveForResourcesById( - activityReportObjectiveOne.id, + activityReportOneObjectiveOne.id, + [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], + ); + + // Report 1 - Activity Report Objective 2 + [activityReportOneObjectiveTwo] = await ActivityReportObjective.findOrCreate({ + where: { + activityReportId: reportOne.id, + status: 'Complete', + objectiveId: objectiveTwo.id, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveTwo.id, + topicId: coachingTopicId, + }, + }); + + await processActivityReportObjectiveForResourcesById( + activityReportOneObjectiveTwo.id, [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], ); @@ -215,9 +309,17 @@ describe('Resources dashboard', () => { // Report 2 ECLKC Resource 1. await processActivityReportObjectiveForResourcesById( activityReportObjectiveTwo.id, - [ECLKC_RESOURCE_URL], + [ECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], ); + // Report 2 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveTwo.id, + topicId: oralHealthTopicId, + }, + }); + // Report 3 (Only Non-ECLKC). const reportThree = await ActivityReport.create({ ...regionOneReportC }); await ActivityRecipient.create({ activityReportId: reportThree.id, grantId: mockGrant.id }); @@ -234,16 +336,54 @@ describe('Resources dashboard', () => { [NONECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], ); + // Report 3 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveThree.id, + topicId: nutritionTopicId, + }, + }); + // Report 4 (No Resources). const reportFour = await ActivityReport.create({ ...regionOneReportD }); await ActivityRecipient.create({ activityReportId: reportFour.id, grantId: mockGrant.id }); - await ActivityReportObjective.create({ + const activityReportObjectiveForReport4 = await ActivityReportObjective.create({ activityReportId: reportFour.id, status: 'Complete', objectiveId: objective.id, }); + // Report 4 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: facilitiesTopicId, + }, + }); + + // Report 4 Topic 2. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: fiscalBudgetTopicId, + }, + }); + + // Report 4 Topic 3. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: erseaTopicId, + }, + }); + + // Report 3 Non-ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportObjectiveForReport4.id, + [ECLKC_RESOURCE_URL2], + ); + // Draft Report (Excluded). const reportDraft = await ActivityReport.create({ ...regionOneDraftReport }); await ActivityRecipient.create({ activityReportId: reportDraft.id, grantId: mockGrant.id }); @@ -260,9 +400,28 @@ describe('Resources dashboard', () => { activityReportObjectiveDraft.id, [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], ); + + // Draft Report 5 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveDraft.id, + topicId: equityTopicId, + }, + }); + + // Draft Report 5 Topic 2. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveDraft.id, + topicId: erseaTopicId, + }, + }); + + arIds = [reportOne.id, reportTwo.id, reportThree.id, reportFour.id, reportDraft.id]; }); afterAll(async () => { + /* const reports = await ActivityReport .findAll({ where: { userId: [mockUser.id] } }); const ids = reports.map((report) => report.id); @@ -271,22 +430,24 @@ describe('Resources dashboard', () => { await ActivityReportObjectiveResource.destroy({ where: { - activityReportObjectiveId: activityReportObjectiveOne.id, + activityReportObjectiveId: activityReportOneObjectiveOne.id, }, }); await ActivityReportObjectiveTopic.destroy({ where: { - activityReportObjectiveId: activityReportObjectiveOne.id, + activityReportObjectiveId: arIds, }, }); - await ActivityReportObjective.destroy({ where: { objectiveId: objective.id } }); + // eslint-disable-next-line max-len + await ActivityReportObjective.destroy({ where: { objectiveId: [objective.id, objectiveTwo.id] } }); await ActivityReport.destroy({ where: { id: ids } }); - await Objective.destroy({ where: { id: objective.id }, force: true }); - await Goal.destroy({ where: { id: goal.id }, force: true }); + await Objective.destroy({ where: { id: [objective.id, objectiveTwo.id] }, force: true }); + await Goal.destroy({ where: { id: [goal.id, goalTwo.id] }, force: true }); await Grant.destroy({ where: { id: GRANT_ID_ONE }, individualHooks: true }); await User.destroy({ where: { id: [mockUser.id] } }); await Recipient.destroy({ where: { id: RECIPIENT_ID } }); + */ await db.sequelize.close(); }); @@ -303,6 +464,7 @@ describe('Resources dashboard', () => { }); const res = await resourceList(scopes); + console.log('\n\n\n---- Resource List: ', res); expect(res.length).toBe(4); expect(res[0].name).toBe(ECLKC_RESOURCE_URL); @@ -550,4 +712,52 @@ describe('Resources dashboard', () => { ], }); }); + + it('flatResources', async () => { + const scopes = await filtersToScopes({}); + const data = await resourceFlatData(scopes); + expect(true).toBe(true); + }); + + it('resourceUseFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { resourceUseResult } = await resourceFlatData(scopes); + expect(resourceUseResult).toBeDefined(); + expect(resourceUseResult.length).toBe(3); + console.log('\n\n\n-----resourceUseResult: ', resourceUseResult); + + expect(resourceUseResult).toStrictEqual([ + { + url: 'https://eclkc.ohs.acf.hhs.gov/test', + rollUpDate: 'Jan-21', + resourceCount: '2', + }, + { + url: 'https://eclkc.ohs.acf.hhs.gov/test2', + rollUpDate: 'Jan-21', + resourceCount: '3', + }, + { + url: 'https://non.test1.gov/a/b/c', + rollUpDate: 'Jan-21', + resourceCount: '2', + }, + ]); + }); + + it('resourceTopicUseFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { topicUseResult } = await resourceFlatData(scopes); + expect(topicUseResult).toBeDefined(); + + expect(topicUseResult).toStrictEqual([ + { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2' }, + { name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4' }, + { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3' }, + { name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2' }, + { name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2' }, + ]); + }); }); diff --git a/src/services/users.js b/src/services/users.js index 3611b33c56..4b338babfb 100644 --- a/src/services/users.js +++ b/src/services/users.js @@ -94,6 +94,7 @@ export async function userEmailIsVerifiedByUserId(userId) { /* Get Statistics by User */ export async function statisticsByUser(user, regions, readonly = false, reportIds = []) { + // Get days joined. const dateJoined = new Date(user.createdAt); const todaysDate = new Date(); From e9fd200f0b0a0bdc7e0e960f04bddf6eedbf190a Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 21 Mar 2024 15:06:35 -0400 Subject: [PATCH 02/75] wip2 --- src/services/dashboards/resource.js | 127 ++++++++++++++--------- src/services/dashboards/resource.test.js | 24 ++++- 2 files changed, 102 insertions(+), 49 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index b88171d52e..817df387a0 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -336,7 +336,7 @@ const switchToTopicCentric = (input) => { If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ export async function resourceFlatData(scopes) { - console.time('flatTime'); + console.time('OVERALLTIME'); // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -362,13 +362,15 @@ export async function resourceFlatData(scopes) { console.timeEnd('scopesTime'); console.log('\n\n\n------ AR IDS from Scopes: ', reportIds); + // Get total number of reports. + const totalReportCount = reportIds.length; + // Write raw sql to generate the flat resource data for the above reportIds. const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; - const createdAroTopicsRolledUpTempTableName = `Z_temp_resource_aro_rolled_up_topics__${uuidv4().replaceAll('-', '_')}`; const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. // Create raw sql to get flat table. @@ -414,6 +416,7 @@ export async function resourceFlatData(scopes) { -- 4.) Create ARO Topics temp table. SELECT ar.id AS "activityReportId", + arot."activityReportObjectiveId", -- We need to group by this incase of multiple aro's. arot."topicId" INTO TEMP ${createdAroTopicsTempTableName} FROM ${createdArTempTableName} ar @@ -421,7 +424,7 @@ export async function resourceFlatData(scopes) { ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveTopics" arot ON aro.id = arot."activityReportObjectiveId" - GROUP BY ar.id, arot."topicId"; + GROUP BY ar.id, arot."activityReportObjectiveId", arot."topicId"; -- 5.) Create Topics temp table (only what we need). SELECT @@ -434,33 +437,21 @@ export async function resourceFlatData(scopes) { FROM ${createdAroTopicsTempTableName} ); - -- 6.) Create Rolled Up Topics temp table (maybe delete this later...). - SELECT - arot."activityReportId", - ARRAY_AGG(DISTINCT arott.name) AS topics - INTO TEMP ${createdAroTopicsRolledUpTempTableName} - FROM ${createdAroTopicsTempTableName} arot - JOIN ${createdTopicsTempTableName} arott - ON arot."topicId" = arott.id - GROUP BY arot."activityReportId"; - - -- 7.) Create Flat Resource temp table. + -- 6.) Create Flat Resource temp table. SELECT ar.id, ar."startDate", ar."rollUpDate", - artog.topics, arorr.domain, arorr.title, - arorr.url + arorr.url, + ar."numberOfParticipants" INTO TEMP ${createdFlatResourceTempTableName} FROM ${createdArTempTableName} ar JOIN ${createdAroResourcesTempTableName} aror ON ar.id = aror."activityReportId" JOIN ${createdResourcesTempTableName} arorr ON aror."resourceId" = arorr.id - LEFT JOIN ${createdAroTopicsRolledUpTempTableName} artog - ON ar.id = artog."activityReportId"; `; console.log('\n\n\n------- SQL BEFORE: ', flatResourceSql); console.time('sqlTime'); @@ -470,25 +461,11 @@ export async function resourceFlatData(scopes) { // Create base tables. await sequelize.query( flatResourceSql, - { - // raw: true, - // nest: false, - type: QueryTypes.SELECT, - // mapToModel: false, - transaction, - }, - ); - - // Select the Flat table. - /* - let flatTable = sequelize.query( - `SELECT * FROM ${createdFlatResourceTempTableName};`, { type: QueryTypes.SELECT, transaction, }, ); - */ // Get resource use result. let resourceUseResult = sequelize.query( @@ -508,7 +485,7 @@ export async function resourceFlatData(scopes) { }, ); - // Get final topic use result. + // Get topic use result. let topicUseResult = sequelize.query(` SELECT t.name, @@ -526,26 +503,82 @@ export async function resourceFlatData(scopes) { transaction, }); - // [flatTable, resourceUseResult, topicUseResult] = await Promise.all([flatTable, resourceUseResult, topicUseResult]); - [resourceUseResult, topicUseResult] = await Promise.all([resourceUseResult, topicUseResult]); + /* Overview */ + // 1.) Participants + let numberOfParticipants = sequelize.query(` + WITH ar_participants AS ( + SELECT + id, + "numberOfParticipants" + FROM ${createdFlatResourceTempTableName} f + GROUP BY id, "numberOfParticipants" + ) + SELECT + SUM("numberOfParticipants") AS participants + FROM ar_participants; + `, { + type: QueryTypes.SELECT, + transaction, + }); + + // 2.) Recipients. + let numberOfRecipients = sequelize.query(` + WITH ars AS ( + SELECT + DISTINCT id + FROM ${createdFlatResourceTempTableName} f + ), recipients AS ( + SELECT + DISTINCT r.id + FROM ars ar + JOIN "ActivityRecipients" arr + ON ar.id = arr."activityReportId" + JOIN "Grants" g + ON arr."grantId" = g.id + JOIN "Recipients" r + ON g."recipientId" = r.id + ) + SELECT + count(r.id) + FROM recipients r; + `, { + type: QueryTypes.SELECT, + transaction, + }); + + // 3.) Reports with Resources. + let pctOfReportsWithResources = sequelize.query(` + SELECT + count(DISTINCT "activityReportId") AS "reportsWithResourcesCount", + ${totalReportCount} AS "totalReportsCount", + CASE WHEN ${totalReportCount} = 0 THEN + 0 + ELSE + (count(DISTINCT "activityReportId")::int / ${totalReportCount}) * 100 + END AS "resourcesPct" + FROM ${createdAroResourcesTempTableName}; + `, { + type: QueryTypes.SELECT, + transaction, + }); + + [resourceUseResult, topicUseResult, numberOfParticipants, numberOfRecipients, pctOfReportsWithResources] = await Promise.all([resourceUseResult, topicUseResult, numberOfParticipants, numberOfRecipients, pctOfReportsWithResources]); // Commit is required to run the query. transaction.commit(); - // console.log('\n\n\n------- SQL FLAT: ', flatTable); + console.log('\n\n\n------- SQL RESOURCE USE: ', resourceUseResult); console.log('\n\n\n------- SQL TOPIC USE: ', topicUseResult); - /* - const tables = await sequelize.query( - `SELECT "name" FROM "Users" LIMIT 1; - SELECT "id" FROM "ActivityReports" LIMIT 2;`, - { - type: QueryTypes.SELECT, - }, - ); - */ + console.log('\n\n\n------- SQL NUM OF PARTICIPANTS: ', numberOfParticipants); + console.log('\n\n\n------- SQL NUM OF RECIPIENTS: ', numberOfRecipients); + console.log('\n\n\n------- SQL PCT OF REPORTS with RESOURCES: ', pctOfReportsWithResources); + console.timeEnd('sqlTime'); - console.timeEnd('flatTime'); - return { resourceUseResult, topicUseResult }; + console.timeEnd('OVERALLTIME'); + const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources }; + return { + resourceUseResult, topicUseResult, numberOfParticipants, overView, + }; } // collect all resource data from the db filtered via the scopes diff --git a/src/services/dashboards/resource.test.js b/src/services/dashboards/resource.test.js index 6e59a552d1..1345996fe7 100644 --- a/src/services/dashboards/resource.test.js +++ b/src/services/dashboards/resource.test.js @@ -344,7 +344,7 @@ describe('Resources dashboard', () => { }, }); - // Report 4 (No Resources). + // Report 4. const reportFour = await ActivityReport.create({ ...regionOneReportD }); await ActivityRecipient.create({ activityReportId: reportFour.id, grantId: mockGrant.id }); @@ -384,6 +384,24 @@ describe('Resources dashboard', () => { [ECLKC_RESOURCE_URL2], ); + // Report 5 (No resources). + const reportFive = await ActivityReport.create({ ...regionOneReportD }); + await ActivityRecipient.create({ activityReportId: reportFive.id, grantId: mockGrant.id }); + + const activityReportObjectiveForReport5 = await ActivityReportObjective.create({ + activityReportId: reportFive.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report 5 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport5.id, + topicId: facilitiesTopicId, + }, + }); + // Draft Report (Excluded). const reportDraft = await ActivityReport.create({ ...regionOneDraftReport }); await ActivityRecipient.create({ activityReportId: reportDraft.id, grantId: mockGrant.id }); @@ -417,7 +435,7 @@ describe('Resources dashboard', () => { }, }); - arIds = [reportOne.id, reportTwo.id, reportThree.id, reportFour.id, reportDraft.id]; + arIds = [reportOne.id, reportTwo.id, reportThree.id, reportFour.id, reportFive.id, reportDraft.id]; }); afterAll(async () => { @@ -550,9 +568,11 @@ describe('Resources dashboard', () => { const data = await resourcesDashboardOverview(scopes); expect(data).toStrictEqual({ participant: { + // Participants. numParticipants: '44', }, recipient: { + // Recipient's Reached. num: '1', // numEclkc: '1', // numNoResources: '0', From 4893e9682ff7492bdc5a42a93427336415a2592a Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 25 Mar 2024 11:32:44 -0400 Subject: [PATCH 03/75] wip --- src/services/dashboards/resource.js | 8 ++-- src/services/dashboards/resource.test.js | 60 +++++++++++++++++------- 2 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 817df387a0..69084f3f24 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -539,7 +539,7 @@ export async function resourceFlatData(scopes) { ON g."recipientId" = r.id ) SELECT - count(r.id) + count(r.id) AS recipients FROM recipients r; `, { type: QueryTypes.SELECT, @@ -549,12 +549,12 @@ export async function resourceFlatData(scopes) { // 3.) Reports with Resources. let pctOfReportsWithResources = sequelize.query(` SELECT - count(DISTINCT "activityReportId") AS "reportsWithResourcesCount", - ${totalReportCount} AS "totalReportsCount", + count(DISTINCT "activityReportId")::decimal AS "reportsWithResourcesCount", + ${totalReportCount}::decimal AS "totalReportsCount", CASE WHEN ${totalReportCount} = 0 THEN 0 ELSE - (count(DISTINCT "activityReportId")::int / ${totalReportCount}) * 100 + (round(count(DISTINCT "activityReportId")::decimal / ${totalReportCount}::decimal, 4) * 100)::decimal END AS "resourcesPct" FROM ${createdAroResourcesTempTableName}; `, { diff --git a/src/services/dashboards/resource.test.js b/src/services/dashboards/resource.test.js index 1345996fe7..1b40144b86 100644 --- a/src/services/dashboards/resource.test.js +++ b/src/services/dashboards/resource.test.js @@ -233,7 +233,7 @@ describe('Resources dashboard', () => { }, { individualHooks: true, }); - console.log("\n\n\n------ Report 1"); + console.log('\n\n\n------ Report 1'); await ActivityRecipient.findOrCreate({ where: { activityReportId: reportOne.id, grantId: mockGrant.id }, }); @@ -384,23 +384,23 @@ describe('Resources dashboard', () => { [ECLKC_RESOURCE_URL2], ); - // Report 5 (No resources). - const reportFive = await ActivityReport.create({ ...regionOneReportD }); - await ActivityRecipient.create({ activityReportId: reportFive.id, grantId: mockGrant.id }); + // Report 5 (No resources). + const reportFive = await ActivityReport.create({ ...regionOneReportD }); + await ActivityRecipient.create({ activityReportId: reportFive.id, grantId: mockGrant.id }); - const activityReportObjectiveForReport5 = await ActivityReportObjective.create({ - activityReportId: reportFive.id, - status: 'Complete', - objectiveId: objective.id, - }); + const activityReportObjectiveForReport5 = await ActivityReportObjective.create({ + activityReportId: reportFive.id, + status: 'Complete', + objectiveId: objective.id, + }); - // Report 5 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveForReport5.id, - topicId: facilitiesTopicId, - }, - }); + // Report 5 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport5.id, + topicId: facilitiesTopicId, + }, + }); // Draft Report (Excluded). const reportDraft = await ActivityReport.create({ ...regionOneDraftReport }); @@ -439,7 +439,6 @@ describe('Resources dashboard', () => { }); afterAll(async () => { - /* const reports = await ActivityReport .findAll({ where: { userId: [mockUser.id] } }); const ids = reports.map((report) => report.id); @@ -465,7 +464,6 @@ describe('Resources dashboard', () => { await Grant.destroy({ where: { id: GRANT_ID_ONE }, individualHooks: true }); await User.destroy({ where: { id: [mockUser.id] } }); await Recipient.destroy({ where: { id: RECIPIENT_ID } }); - */ await db.sequelize.close(); }); @@ -780,4 +778,30 @@ describe('Resources dashboard', () => { { name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2' }, ]); }); + + it('overviewFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { overView } = await resourceFlatData(scopes); + expect(overView).toBeDefined(); + const { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources } = overView; + + // Number of Participants. + expect(numberOfParticipants).toStrictEqual([{ + participants: '44', + }]); + + // Number of Recipients. + expect(numberOfRecipients).toStrictEqual([{ + recipients: '1', + }]); + + // Percent of Reports with Resources. + expect(pctOfReportsWithResources).toStrictEqual([ + { + reportsWithResourcesCount: '4', + totalReportsCount: '5', + resourcesPct: '80.0000', + }, + ]); + }); }); From c8dc05a1a155337cc4618235b7a80239b91336b8 Mon Sep 17 00:00:00 2001 From: nvms Date: Mon, 25 Mar 2024 12:28:30 -0400 Subject: [PATCH 04/75] first pass --- frontend/src/components/MultiSelect.js | 63 ++++++++++--------- .../Pages/__tests__/activitySummary.js | 21 +++++++ .../ActivityReport/Pages/activitySummary.js | 15 ++++- 3 files changed, 69 insertions(+), 30 deletions(-) diff --git a/frontend/src/components/MultiSelect.js b/frontend/src/components/MultiSelect.js index a774d9de27..b677507ba1 100644 --- a/frontend/src/components/MultiSelect.js +++ b/frontend/src/components/MultiSelect.js @@ -104,6 +104,7 @@ function MultiSelect({ onCreateOption, placeholderText, components: componentReplacements, + onClick = () => {}, }) { const inputId = `select-${uuidv4()}`; @@ -165,35 +166,37 @@ function MultiSelect({ render={({ onChange: controllerOnChange, value, onBlur }) => { const values = value ? getValues(value) : value; return ( - { - if (onItemSelected) { - onItemSelected(event); - } else if (event) { - onChange(event, controllerOnChange); - } else { - controllerOnChange([]); - } - }} - inputId={inputId} - styles={styles(singleRowInput)} - components={{ ...componentReplacements, DropdownIndicator }} - options={options} - isDisabled={disabled} - tabSelectsValue={false} - isClearable={multiSelectOptions.isClearable} - closeMenuOnSelect={multiSelectOptions.closeMenuOnSelect || false} - controlShouldRenderValue={multiSelectOptions.controlShouldRenderValue} - hideSelectedOptions={multiSelectOptions.hideSelectedOptions} - placeholder={placeholderText || ''} - onCreateOption={onCreateOption} - isMulti - required={!!(required)} - /> +
{}} role="button" tabIndex="0" data-testid={`${name}-click-container`}> + { + if (onItemSelected) { + onItemSelected(event); + } else if (event) { + onChange(event, controllerOnChange); + } else { + controllerOnChange([]); + } + }} + inputId={inputId} + styles={styles(singleRowInput)} + components={{ ...componentReplacements, DropdownIndicator }} + options={options} + isDisabled={disabled} + tabSelectsValue={false} + isClearable={multiSelectOptions.isClearable} + closeMenuOnSelect={multiSelectOptions.closeMenuOnSelect || false} + controlShouldRenderValue={multiSelectOptions.controlShouldRenderValue} + hideSelectedOptions={multiSelectOptions.hideSelectedOptions} + placeholder={placeholderText || ''} + onCreateOption={onCreateOption} + isMulti + required={!!(required)} + /> +
); }} control={control} @@ -253,6 +256,7 @@ MultiSelect.propTypes = { }), required: PropTypes.oneOfType([PropTypes.string, PropTypes.bool]), placeholderText: PropTypes.string, + onClick: PropTypes.func, }; MultiSelect.defaultProps = { @@ -269,6 +273,7 @@ MultiSelect.defaultProps = { onItemSelected: null, onCreateOption: null, placeholderText: null, + onClick: null, }; export default MultiSelect; diff --git a/frontend/src/pages/ActivityReport/Pages/__tests__/activitySummary.js b/frontend/src/pages/ActivityReport/Pages/__tests__/activitySummary.js index 74e3e06640..a9ae3da6ff 100644 --- a/frontend/src/pages/ActivityReport/Pages/__tests__/activitySummary.js +++ b/frontend/src/pages/ActivityReport/Pages/__tests__/activitySummary.js @@ -66,6 +66,27 @@ describe('activity summary', () => { expect(await screen.findByText('Duration must be less than or equal to 99 hours')).toBeInTheDocument(); }); }); + + describe('activity recipients validation', () => { + it('shows a validation message when clicked and recipient type is not selected', async () => { + render(); + const input = screen.getByTestId('activityRecipients-click-container'); + userEvent.click(input); + expect(await screen.findByText('You must first select who the activity is for')).toBeInTheDocument(); + }); + + it('hides the message when the recipient type is selected', async () => { + const { container } = render(); + const input = screen.getByTestId('activityRecipients-click-container'); + userEvent.click(input); + expect(await screen.findByText('You must first select who the activity is for')).toBeInTheDocument(); + await act(() => { + const recipient = container.querySelector('#category-recipient'); + userEvent.click(recipient); + }); + expect(screen.queryByText('You must first select who the activity is for')).not.toBeInTheDocument(); + }); + }); }); describe('groups', () => { diff --git a/frontend/src/pages/ActivityReport/Pages/activitySummary.js b/frontend/src/pages/ActivityReport/Pages/activitySummary.js index 261cd72ed0..f1e325e576 100644 --- a/frontend/src/pages/ActivityReport/Pages/activitySummary.js +++ b/frontend/src/pages/ActivityReport/Pages/activitySummary.js @@ -55,11 +55,13 @@ const ActivitySummary = ({ setValue, control, getValues, + clearErrors, } = useFormContext(); const [useGroup, setUseGroup] = useState(false); const [showGroupInfo, setShowGroupInfo] = useState(false); const [groupRecipientIds, setGroupRecipientIds] = useState([]); + const [shouldValidateActivityRecipients, setShouldValidateActivityRecipients] = useState(false); const activityRecipientType = watch('activityRecipientType'); const watchFormRecipients = watch('activityRecipients'); @@ -193,6 +195,16 @@ const ActivitySummary = ({ /> ); + useEffect(() => { + if (!shouldValidateActivityRecipients) return; + + if (disableRecipients) { + setValue('activityRecipients', [], { shouldValidate: true }); + } else { + clearErrors('activityRecipients'); + } + }, [disableRecipients, shouldValidateActivityRecipients, setValue, clearErrors]); + const renderRecipients = (marginTop = 2, marginBottom = 0) => (
{!disableRecipients @@ -211,9 +223,10 @@ const ActivitySummary = ({ valueProperty="activityRecipientId" labelProperty="name" simple={false} - required="Select at least one" + required={disableRecipients ? 'You must first select who the activity is for' : 'Select at least one'} options={selectedRecipients} placeholderText={placeholderText} + onClick={() => setShouldValidateActivityRecipients(true)} />
From 8fb316a5335d03d92b9e1296669bfdeae93b9fd5 Mon Sep 17 00:00:00 2001 From: nvms Date: Mon, 25 Mar 2024 12:52:57 -0400 Subject: [PATCH 05/75] maybe make the e2e test happy --- frontend/src/components/MultiSelect.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/MultiSelect.js b/frontend/src/components/MultiSelect.js index b677507ba1..992ad4c975 100644 --- a/frontend/src/components/MultiSelect.js +++ b/frontend/src/components/MultiSelect.js @@ -166,7 +166,8 @@ function MultiSelect({ render={({ onChange: controllerOnChange, value, onBlur }) => { const values = value ? getValues(value) : value; return ( -
{}} role="button" tabIndex="0" data-testid={`${name}-click-container`}> + // eslint-disable-next-line jsx-a11y/no-static-element-interactions +
{}} data-testid={`${name}-click-container`}> Date: Mon, 25 Mar 2024 14:41:19 -0400 Subject: [PATCH 06/75] completed tests --- src/lib/cache.ts | 2 +- src/services/dashboards/resource.js | 69 ++- src/services/dashboards/resource.test.js | 300 +--------- src/services/dashboards/resourceFlat.test.js | 548 +++++++++++++++++++ src/services/users.js | 1 - 5 files changed, 621 insertions(+), 299 deletions(-) create mode 100644 src/services/dashboards/resourceFlat.test.js diff --git a/src/lib/cache.ts b/src/lib/cache.ts index e724457d7f..d62450df90 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (false) { + if (!ignoreCache) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 69084f3f24..0088337019 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -336,13 +336,12 @@ const switchToTopicCentric = (input) => { If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ export async function resourceFlatData(scopes) { - console.time('OVERALLTIME'); // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; // Get all ActivityReport ID's using SCOPES. // We don't want to write custom filters. - console.time('scopesTime'); + const reportIds = await ActivityReport.findAll({ attributes: [ 'id', @@ -359,8 +358,6 @@ export async function resourceFlatData(scopes) { }, raw: true, }); - console.timeEnd('scopesTime'); - console.log('\n\n\n------ AR IDS from Scopes: ', reportIds); // Get total number of reports. const totalReportCount = reportIds.length; @@ -453,8 +450,7 @@ export async function resourceFlatData(scopes) { JOIN ${createdResourcesTempTableName} arorr ON aror."resourceId" = arorr.id `; - console.log('\n\n\n------- SQL BEFORE: ', flatResourceSql); - console.time('sqlTime'); + // await sequelize.query('SELECT * FROM projects', { raw: true }); const transaction = await sequelize.transaction(); @@ -562,20 +558,57 @@ export async function resourceFlatData(scopes) { transaction, }); - [resourceUseResult, topicUseResult, numberOfParticipants, numberOfRecipients, pctOfReportsWithResources] = await Promise.all([resourceUseResult, topicUseResult, numberOfParticipants, numberOfRecipients, pctOfReportsWithResources]); + // 4.) ECKLKC resource percentage. + let pctOfECKLKCResources = sequelize.query(` + WITH eclkc AS ( + SELECT + COUNT(DISTINCT url) AS "eclkcCount" + FROM ${createdFlatResourceTempTableName} + WHERE url ilike '%eclkc.ohs.acf.hhs.gov%' + ), allres AS ( + SELECT + COUNT(DISTINCT url) AS "allCount" + FROM ${createdFlatResourceTempTableName} + ) + SELECT + e."eclkcCount", + r."allCount", + CASE WHEN + r."allCount" = 0 + THEN 0 + ELSE round(e."eclkcCount" / r."allCount"::decimal * 100,4) + END AS "eclkcPct" + FROM eclkc e + JOIN allres r + ON 1=1; + `, { + type: QueryTypes.SELECT, + transaction, + }); + + [ + resourceUseResult, + topicUseResult, + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources] = await Promise.all( + [ + resourceUseResult, + topicUseResult, + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources, + ], + ); // Commit is required to run the query. transaction.commit(); - console.log('\n\n\n------- SQL RESOURCE USE: ', resourceUseResult); - console.log('\n\n\n------- SQL TOPIC USE: ', topicUseResult); - console.log('\n\n\n------- SQL NUM OF PARTICIPANTS: ', numberOfParticipants); - console.log('\n\n\n------- SQL NUM OF RECIPIENTS: ', numberOfRecipients); - console.log('\n\n\n------- SQL PCT OF REPORTS with RESOURCES: ', pctOfReportsWithResources); - - console.timeEnd('sqlTime'); - console.timeEnd('OVERALLTIME'); - const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources }; + const overView = { + numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, + }; return { resourceUseResult, topicUseResult, numberOfParticipants, overView, }; @@ -594,7 +627,6 @@ export async function resourceData(scopes, skipResources = false, skipTopics = f viaObjectives: null, viaGoals: null, }; - console.time('reportsTime2'); dbData.allReports = await ActivityReport.findAll({ attributes: [ 'id', @@ -653,7 +685,6 @@ export async function resourceData(scopes, skipResources = false, skipTopics = f raw: true, }); - console.timeEnd('reportsTime2'); [ // dbData.allReports, // dbData.viaReport, @@ -1840,7 +1871,6 @@ export async function resourceTopicUse(scopes) { } export async function resourceDashboardPhase1(scopes) { - console.log('\n\n\n------Phase1'); const data = await resourceData(scopes); return { overview: generateResourcesDashboardOverview(data), @@ -1851,7 +1881,6 @@ export async function resourceDashboardPhase1(scopes) { } export async function resourceDashboard(scopes) { - console.log('\n\n\n------Old'); const data = await resourceData(scopes); return { overview: generateResourcesDashboardOverview(data), diff --git a/src/services/dashboards/resource.test.js b/src/services/dashboards/resource.test.js index 1b40144b86..9490c337a2 100644 --- a/src/services/dashboards/resource.test.js +++ b/src/services/dashboards/resource.test.js @@ -21,7 +21,6 @@ import { resourceUse, resourceDashboard, resourceTopicUse, - resourceFlatData, } from './resource'; import { RESOURCE_DOMAIN } from '../../constants'; import { processActivityReportObjectiveForResourcesById } from '../resource'; @@ -88,7 +87,7 @@ const reportObject = { targetPopulations: ['pop'], reason: ['reason'], participants: ['participants'], - // topics: ['Coaching'], + topics: ['Coaching'], ttaType: ['technical-assistance'], version: 2, }; @@ -99,7 +98,7 @@ const regionOneReportA = { duration: 1, startDate: '2021-01-02T12:00:00Z', endDate: '2021-01-31T12:00:00Z', - // topics: ['Coaching', 'ERSEA'], + topics: ['Coaching', 'ERSEA'], }; const regionOneReportB = { @@ -108,7 +107,7 @@ const regionOneReportB = { duration: 2, startDate: '2021-01-15T12:00:00Z', endDate: '2021-02-15T12:00:00Z', - // topics: ['Oral Health'], + topics: ['Oral Health'], }; const regionOneReportC = { @@ -117,7 +116,7 @@ const regionOneReportC = { duration: 3, startDate: '2021-01-20T12:00:00Z', endDate: '2021-02-28T12:00:00Z', - // topics: ['Nutrition'], + topics: ['Nutrition'], }; const regionOneReportD = { @@ -126,7 +125,7 @@ const regionOneReportD = { duration: 3, startDate: '2021-01-22T12:00:00Z', endDate: '2021-01-31T12:00:00Z', - // topics: ['Facilities', 'Fiscal / Budget', 'ERSEA'], + topics: ['Facilities', 'Fiscal / Budget', 'ERSEA'], }; const regionOneDraftReport = { @@ -137,19 +136,15 @@ const regionOneDraftReport = { endDate: '2021-01-31T12:00:00Z', submissionStatus: REPORT_STATUSES.DRAFT, calculatedStatus: REPORT_STATUSES.DRAFT, - // topics: ['Equity', 'ERSEA'], + topics: ['Equity', 'ERSEA'], }; let grant; let goal; let objective; -let goalTwo; -let objectiveTwo; -let activityReportOneObjectiveOne; -let activityReportOneObjectiveTwo; +let activityReportObjectiveOne; let activityReportObjectiveTwo; let activityReportObjectiveThree; -let arIds; describe('Resources dashboard', () => { beforeAll(async () => { @@ -161,7 +156,6 @@ describe('Resources dashboard', () => { individualHooks: true, }); [goal] = await Goal.findOrCreate({ where: mockGoal, validate: true, individualHooks: true }); - [goalTwo] = await Goal.findOrCreate({ where: { ...mockGoal, name: 'Goal 2' }, validate: true, individualHooks: true }); [objective] = await Objective.findOrCreate({ where: { title: 'Objective 1', @@ -170,76 +164,17 @@ describe('Resources dashboard', () => { }, }); - [objectiveTwo] = await Objective.findOrCreate({ - where: { - title: 'Objective 2', - goalId: goalTwo.dataValues.id, - status: 'In Progress', - }, - }); - - // Get topic ID's. - const { topicId: classOrgTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'CLASS: Classroom Organization' }, - raw: true, - }); - - const { topicId: erseaTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'ERSEA' }, - raw: true, - }); - - const { topicId: coachingTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Coaching' }, - raw: true, - }); - - const { topicId: facilitiesTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Facilities' }, - raw: true, - }); - - const { topicId: fiscalBudgetTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Fiscal / Budget' }, - raw: true, - }); - - const { topicId: nutritionTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Nutrition' }, - raw: true, - }); - - const { topicId: oralHealthTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Oral Health' }, - raw: true, - }); - - const { topicId: equityTopicId } = await Topic.findOne({ - attributes: [['id', 'topicId']], - where: { name: 'Equity' }, - raw: true, - }); - // Report 1 (Mixed Resources). const reportOne = await ActivityReport.create({ ...regionOneReportA, }, { individualHooks: true, }); - console.log('\n\n\n------ Report 1'); await ActivityRecipient.findOrCreate({ where: { activityReportId: reportOne.id, grantId: mockGrant.id }, }); - // Report 1 - Activity Report Objective 1 - [activityReportOneObjectiveOne] = await ActivityReportObjective.findOrCreate({ + [activityReportObjectiveOne] = await ActivityReportObjective.findOrCreate({ where: { activityReportId: reportOne.id, status: 'Complete', @@ -247,52 +182,23 @@ describe('Resources dashboard', () => { }, }); - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportOneObjectiveOne.id, - topicId: classOrgTopicId, - }, - }); - - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportOneObjectiveOne.id, - topicId: erseaTopicId, - }, + const { topicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'CLASS: Classroom Organization' }, + raw: true, }); await ActivityReportObjectiveTopic.findOrCreate({ where: { - activityReportObjectiveId: activityReportOneObjectiveOne.id, - topicId: coachingTopicId, + activityReportObjectiveId: activityReportObjectiveOne.id, + topicId, }, }); // Report 1 ECLKC Resource 1. // Report 1 Non-ECLKC Resource 1. await processActivityReportObjectiveForResourcesById( - activityReportOneObjectiveOne.id, - [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], - ); - - // Report 1 - Activity Report Objective 2 - [activityReportOneObjectiveTwo] = await ActivityReportObjective.findOrCreate({ - where: { - activityReportId: reportOne.id, - status: 'Complete', - objectiveId: objectiveTwo.id, - }, - }); - - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportOneObjectiveTwo.id, - topicId: coachingTopicId, - }, - }); - - await processActivityReportObjectiveForResourcesById( - activityReportOneObjectiveTwo.id, + activityReportObjectiveOne.id, [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], ); @@ -309,17 +215,9 @@ describe('Resources dashboard', () => { // Report 2 ECLKC Resource 1. await processActivityReportObjectiveForResourcesById( activityReportObjectiveTwo.id, - [ECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], + [ECLKC_RESOURCE_URL], ); - // Report 2 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveTwo.id, - topicId: oralHealthTopicId, - }, - }); - // Report 3 (Only Non-ECLKC). const reportThree = await ActivityReport.create({ ...regionOneReportC }); await ActivityRecipient.create({ activityReportId: reportThree.id, grantId: mockGrant.id }); @@ -336,72 +234,16 @@ describe('Resources dashboard', () => { [NONECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], ); - // Report 3 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveThree.id, - topicId: nutritionTopicId, - }, - }); - - // Report 4. + // Report 4 (No Resources). const reportFour = await ActivityReport.create({ ...regionOneReportD }); await ActivityRecipient.create({ activityReportId: reportFour.id, grantId: mockGrant.id }); - const activityReportObjectiveForReport4 = await ActivityReportObjective.create({ + await ActivityReportObjective.create({ activityReportId: reportFour.id, status: 'Complete', objectiveId: objective.id, }); - // Report 4 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveForReport4.id, - topicId: facilitiesTopicId, - }, - }); - - // Report 4 Topic 2. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveForReport4.id, - topicId: fiscalBudgetTopicId, - }, - }); - - // Report 4 Topic 3. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveForReport4.id, - topicId: erseaTopicId, - }, - }); - - // Report 3 Non-ECLKC Resource 1. - await processActivityReportObjectiveForResourcesById( - activityReportObjectiveForReport4.id, - [ECLKC_RESOURCE_URL2], - ); - - // Report 5 (No resources). - const reportFive = await ActivityReport.create({ ...regionOneReportD }); - await ActivityRecipient.create({ activityReportId: reportFive.id, grantId: mockGrant.id }); - - const activityReportObjectiveForReport5 = await ActivityReportObjective.create({ - activityReportId: reportFive.id, - status: 'Complete', - objectiveId: objective.id, - }); - - // Report 5 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveForReport5.id, - topicId: facilitiesTopicId, - }, - }); - // Draft Report (Excluded). const reportDraft = await ActivityReport.create({ ...regionOneDraftReport }); await ActivityRecipient.create({ activityReportId: reportDraft.id, grantId: mockGrant.id }); @@ -418,24 +260,6 @@ describe('Resources dashboard', () => { activityReportObjectiveDraft.id, [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], ); - - // Draft Report 5 Topic 1. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveDraft.id, - topicId: equityTopicId, - }, - }); - - // Draft Report 5 Topic 2. - await ActivityReportObjectiveTopic.findOrCreate({ - where: { - activityReportObjectiveId: activityReportObjectiveDraft.id, - topicId: erseaTopicId, - }, - }); - - arIds = [reportOne.id, reportTwo.id, reportThree.id, reportFour.id, reportFive.id, reportDraft.id]; }); afterAll(async () => { @@ -447,20 +271,19 @@ describe('Resources dashboard', () => { await ActivityReportObjectiveResource.destroy({ where: { - activityReportObjectiveId: activityReportOneObjectiveOne.id, + activityReportObjectiveId: activityReportObjectiveOne.id, }, }); await ActivityReportObjectiveTopic.destroy({ where: { - activityReportObjectiveId: arIds, + activityReportObjectiveId: activityReportObjectiveOne.id, }, }); - // eslint-disable-next-line max-len - await ActivityReportObjective.destroy({ where: { objectiveId: [objective.id, objectiveTwo.id] } }); + await ActivityReportObjective.destroy({ where: { objectiveId: objective.id } }); await ActivityReport.destroy({ where: { id: ids } }); - await Objective.destroy({ where: { id: [objective.id, objectiveTwo.id] }, force: true }); - await Goal.destroy({ where: { id: [goal.id, goalTwo.id] }, force: true }); + await Objective.destroy({ where: { id: objective.id }, force: true }); + await Goal.destroy({ where: { id: goal.id }, force: true }); await Grant.destroy({ where: { id: GRANT_ID_ONE }, individualHooks: true }); await User.destroy({ where: { id: [mockUser.id] } }); await Recipient.destroy({ where: { id: RECIPIENT_ID } }); @@ -480,7 +303,6 @@ describe('Resources dashboard', () => { }); const res = await resourceList(scopes); - console.log('\n\n\n---- Resource List: ', res); expect(res.length).toBe(4); expect(res[0].name).toBe(ECLKC_RESOURCE_URL); @@ -566,11 +388,9 @@ describe('Resources dashboard', () => { const data = await resourcesDashboardOverview(scopes); expect(data).toStrictEqual({ participant: { - // Participants. numParticipants: '44', }, recipient: { - // Recipient's Reached. num: '1', // numEclkc: '1', // numNoResources: '0', @@ -730,78 +550,4 @@ describe('Resources dashboard', () => { ], }); }); - - it('flatResources', async () => { - const scopes = await filtersToScopes({}); - const data = await resourceFlatData(scopes); - expect(true).toBe(true); - }); - - it('resourceUseFlat', async () => { - const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { resourceUseResult } = await resourceFlatData(scopes); - expect(resourceUseResult).toBeDefined(); - expect(resourceUseResult.length).toBe(3); - console.log('\n\n\n-----resourceUseResult: ', resourceUseResult); - - expect(resourceUseResult).toStrictEqual([ - { - url: 'https://eclkc.ohs.acf.hhs.gov/test', - rollUpDate: 'Jan-21', - resourceCount: '2', - }, - { - url: 'https://eclkc.ohs.acf.hhs.gov/test2', - rollUpDate: 'Jan-21', - resourceCount: '3', - }, - { - url: 'https://non.test1.gov/a/b/c', - rollUpDate: 'Jan-21', - resourceCount: '2', - }, - ]); - }); - - it('resourceTopicUseFlat', async () => { - const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { topicUseResult } = await resourceFlatData(scopes); - expect(topicUseResult).toBeDefined(); - - expect(topicUseResult).toStrictEqual([ - { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2' }, - { name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4' }, - { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3' }, - { name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2' }, - { name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2' }, - ]); - }); - - it('overviewFlat', async () => { - const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { overView } = await resourceFlatData(scopes); - expect(overView).toBeDefined(); - const { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources } = overView; - - // Number of Participants. - expect(numberOfParticipants).toStrictEqual([{ - participants: '44', - }]); - - // Number of Recipients. - expect(numberOfRecipients).toStrictEqual([{ - recipients: '1', - }]); - - // Percent of Reports with Resources. - expect(pctOfReportsWithResources).toStrictEqual([ - { - reportsWithResourcesCount: '4', - totalReportsCount: '5', - resourcesPct: '80.0000', - }, - ]); - }); }); diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js new file mode 100644 index 0000000000..86704e9ee8 --- /dev/null +++ b/src/services/dashboards/resourceFlat.test.js @@ -0,0 +1,548 @@ +import { REPORT_STATUSES } from '@ttahub/common'; +import db, { + ActivityReport, + ActivityRecipient, + Topic, + User, + Recipient, + Grant, + NextStep, + Goal, + Objective, + ActivityReportObjective, + ActivityReportObjectiveResource, + ActivityReportObjectiveTopic, +} from '../../models'; +import filtersToScopes from '../../scopes'; +import { + resourceFlatData, +} from './resource'; +import { RESOURCE_DOMAIN } from '../../constants'; +import { processActivityReportObjectiveForResourcesById } from '../resource'; + +const RECIPIENT_ID = 46204400; +const GRANT_ID_ONE = 107843; +const REGION_ID = 14; +const NONECLKC_DOMAIN = 'non.test1.gov'; +const ECLKC_RESOURCE_URL = `https://${RESOURCE_DOMAIN.ECLKC}/test`; +const ECLKC_RESOURCE_URL2 = `https://${RESOURCE_DOMAIN.ECLKC}/test2`; +const NONECLKC_RESOURCE_URL = `https://${NONECLKC_DOMAIN}/a/b/c`; + +const mockUser = { + id: 5426871, + homeRegionId: 1, + name: 'user5426862', + hsesUsername: 'user5426862', + hsesUserId: '5426862', + lastLogin: new Date(), +}; + +const mockRecipient = { + name: 'recipient', + id: RECIPIENT_ID, + uei: 'NNA5N2KHMGN2XX', +}; + +const mockGrant = { + id: GRANT_ID_ONE, + number: `${GRANT_ID_ONE}`, + recipientId: RECIPIENT_ID, + regionId: REGION_ID, + status: 'Active', +}; + +const mockGoal = { + name: 'Goal 1', + status: 'Draft', + endDate: null, + isFromSmartsheetTtaPlan: false, + onApprovedAR: false, + onAR: false, + grantId: GRANT_ID_ONE, + createdVia: 'rtr', +}; + +const reportObject = { + activityRecipientType: 'recipient', + submissionStatus: REPORT_STATUSES.SUBMITTED, + calculatedStatus: REPORT_STATUSES.APPROVED, + userId: mockUser.id, + lastUpdatedById: mockUser.id, + ECLKCResourcesUsed: ['test'], + activityRecipients: [ + { grantId: GRANT_ID_ONE }, + ], + approvingManagerId: 1, + numberOfParticipants: 11, + deliveryMethod: 'method', + duration: 1, + endDate: '2000-01-01T12:00:00Z', + startDate: '2000-01-01T12:00:00Z', + requester: 'requester', + targetPopulations: ['pop'], + reason: ['reason'], + participants: ['participants'], + ttaType: ['technical-assistance'], + version: 2, +}; + +const regionOneReportA = { + ...reportObject, + regionId: REGION_ID, + duration: 1, + startDate: '2021-01-02T12:00:00Z', + endDate: '2021-01-31T12:00:00Z', +}; + +const regionOneReportB = { + ...reportObject, + regionId: REGION_ID, + duration: 2, + startDate: '2021-01-15T12:00:00Z', + endDate: '2021-02-15T12:00:00Z', +}; + +const regionOneReportC = { + ...reportObject, + regionId: REGION_ID, + duration: 3, + startDate: '2021-01-20T12:00:00Z', + endDate: '2021-02-28T12:00:00Z', +}; + +const regionOneReportD = { + ...reportObject, + regionId: REGION_ID, + duration: 3, + startDate: '2021-01-22T12:00:00Z', + endDate: '2021-01-31T12:00:00Z', +}; + +const regionOneDraftReport = { + ...reportObject, + regionId: REGION_ID, + duration: 7, + startDate: '2021-01-02T12:00:00Z', + endDate: '2021-01-31T12:00:00Z', + submissionStatus: REPORT_STATUSES.DRAFT, + calculatedStatus: REPORT_STATUSES.DRAFT, +}; + +let grant; +let goal; +let objective; +let goalTwo; +let objectiveTwo; +let activityReportOneObjectiveOne; +let activityReportOneObjectiveTwo; +let activityReportObjectiveTwo; +let activityReportObjectiveThree; +let arIds; + +describe('Resources dashboard', () => { + beforeAll(async () => { + await User.findOrCreate({ where: mockUser, individualHooks: true }); + await Recipient.findOrCreate({ where: mockRecipient, individualHooks: true }); + [grant] = await Grant.findOrCreate({ + where: mockGrant, + validate: true, + individualHooks: true, + }); + [goal] = await Goal.findOrCreate({ where: mockGoal, validate: true, individualHooks: true }); + [goalTwo] = await Goal.findOrCreate({ where: { ...mockGoal, name: 'Goal 2' }, validate: true, individualHooks: true }); + [objective] = await Objective.findOrCreate({ + where: { + title: 'Objective 1', + goalId: goal.dataValues.id, + status: 'In Progress', + }, + }); + + [objectiveTwo] = await Objective.findOrCreate({ + where: { + title: 'Objective 2', + goalId: goalTwo.dataValues.id, + status: 'In Progress', + }, + }); + + // Get topic ID's. + const { topicId: classOrgTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'CLASS: Classroom Organization' }, + raw: true, + }); + + const { topicId: erseaTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'ERSEA' }, + raw: true, + }); + + const { topicId: coachingTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Coaching' }, + raw: true, + }); + + const { topicId: facilitiesTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Facilities' }, + raw: true, + }); + + const { topicId: fiscalBudgetTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Fiscal / Budget' }, + raw: true, + }); + + const { topicId: nutritionTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Nutrition' }, + raw: true, + }); + + const { topicId: oralHealthTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Oral Health' }, + raw: true, + }); + + const { topicId: equityTopicId } = await Topic.findOne({ + attributes: [['id', 'topicId']], + where: { name: 'Equity' }, + raw: true, + }); + + // Report 1 (Mixed Resources). + const reportOne = await ActivityReport.create({ + ...regionOneReportA, + }, { + individualHooks: true, + }); + await ActivityRecipient.findOrCreate({ + where: { activityReportId: reportOne.id, grantId: mockGrant.id }, + }); + + // Report 1 - Activity Report Objective 1 + [activityReportOneObjectiveOne] = await ActivityReportObjective.findOrCreate({ + where: { + activityReportId: reportOne.id, + status: 'Complete', + objectiveId: objective.id, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: classOrgTopicId, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: erseaTopicId, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + topicId: coachingTopicId, + }, + }); + + // Report 1 ECLKC Resource 1. + // Report 1 Non-ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportOneObjectiveOne.id, + [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], + ); + + // Report 1 - Activity Report Objective 2 + [activityReportOneObjectiveTwo] = await ActivityReportObjective.findOrCreate({ + where: { + activityReportId: reportOne.id, + status: 'Complete', + objectiveId: objectiveTwo.id, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveTwo.id, + topicId: coachingTopicId, + }, + }); + + await processActivityReportObjectiveForResourcesById( + activityReportOneObjectiveTwo.id, + [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], + ); + + // Report 2 (Only ECLKC). + const reportTwo = await ActivityReport.create({ ...regionOneReportB }); + await ActivityRecipient.create({ activityReportId: reportTwo.id, grantId: mockGrant.id }); + + activityReportObjectiveTwo = await ActivityReportObjective.create({ + activityReportId: reportTwo.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report 2 ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportObjectiveTwo.id, + [ECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], + ); + + // Report 2 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveTwo.id, + topicId: oralHealthTopicId, + }, + }); + + // Report 3 (Only Non-ECLKC). + const reportThree = await ActivityReport.create({ ...regionOneReportC }); + await ActivityRecipient.create({ activityReportId: reportThree.id, grantId: mockGrant.id }); + + activityReportObjectiveThree = await ActivityReportObjective.create({ + activityReportId: reportThree.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report 3 Non-ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportObjectiveThree.id, + [NONECLKC_RESOURCE_URL, ECLKC_RESOURCE_URL2], + ); + + // Report 3 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveThree.id, + topicId: nutritionTopicId, + }, + }); + + // Report 4. + const reportFour = await ActivityReport.create({ ...regionOneReportD }); + await ActivityRecipient.create({ activityReportId: reportFour.id, grantId: mockGrant.id }); + + const activityReportObjectiveForReport4 = await ActivityReportObjective.create({ + activityReportId: reportFour.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report 4 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: facilitiesTopicId, + }, + }); + + // Report 4 Topic 2. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: fiscalBudgetTopicId, + }, + }); + + // Report 4 Topic 3. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport4.id, + topicId: erseaTopicId, + }, + }); + + // Report 3 Non-ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportObjectiveForReport4.id, + [ECLKC_RESOURCE_URL2], + ); + + // Report 5 (No resources). + const reportFive = await ActivityReport.create({ ...regionOneReportD }); + await ActivityRecipient.create({ activityReportId: reportFive.id, grantId: mockGrant.id }); + + const activityReportObjectiveForReport5 = await ActivityReportObjective.create({ + activityReportId: reportFive.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report 5 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveForReport5.id, + topicId: facilitiesTopicId, + }, + }); + + // Draft Report (Excluded). + const reportDraft = await ActivityReport.create({ ...regionOneDraftReport }); + await ActivityRecipient.create({ activityReportId: reportDraft.id, grantId: mockGrant.id }); + + const activityReportObjectiveDraft = await ActivityReportObjective.create({ + activityReportId: reportDraft.id, + status: 'Complete', + objectiveId: objective.id, + }); + + // Report Draft ECLKC Resource 1. + // Report Draft Non-ECLKC Resource 1. + await processActivityReportObjectiveForResourcesById( + activityReportObjectiveDraft.id, + [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], + ); + + // Draft Report 5 Topic 1. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveDraft.id, + topicId: equityTopicId, + }, + }); + + // Draft Report 5 Topic 2. + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportObjectiveDraft.id, + topicId: erseaTopicId, + }, + }); + + arIds = [ + reportOne.id, + reportTwo.id, + reportThree.id, + reportFour.id, + reportFive.id, + reportDraft.id, + ]; + }); + + afterAll(async () => { + const reports = await ActivityReport + .findAll({ where: { userId: [mockUser.id] } }); + const ids = reports.map((report) => report.id); + await NextStep.destroy({ where: { activityReportId: ids } }); + await ActivityRecipient.destroy({ where: { activityReportId: ids } }); + + await ActivityReportObjectiveResource.destroy({ + where: { + activityReportObjectiveId: activityReportOneObjectiveOne.id, + }, + }); + await ActivityReportObjectiveTopic.destroy({ + where: { + activityReportObjectiveId: arIds, + }, + }); + + // eslint-disable-next-line max-len + await ActivityReportObjective.destroy({ where: { objectiveId: [objective.id, objectiveTwo.id] } }); + await ActivityReport.destroy({ where: { id: ids } }); + await Objective.destroy({ where: { id: [objective.id, objectiveTwo.id] }, force: true }); + await Goal.destroy({ where: { id: [goal.id, goalTwo.id] }, force: true }); + await Grant.destroy({ where: { id: GRANT_ID_ONE }, individualHooks: true }); + await User.destroy({ where: { id: [mockUser.id] } }); + await Recipient.destroy({ where: { id: RECIPIENT_ID } }); + await db.sequelize.close(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('resourceUseFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { resourceUseResult } = await resourceFlatData(scopes); + expect(resourceUseResult).toBeDefined(); + expect(resourceUseResult.length).toBe(3); + + expect(resourceUseResult).toStrictEqual([ + { + url: 'https://eclkc.ohs.acf.hhs.gov/test', + rollUpDate: 'Jan-21', + resourceCount: '2', + }, + { + url: 'https://eclkc.ohs.acf.hhs.gov/test2', + rollUpDate: 'Jan-21', + resourceCount: '3', + }, + { + url: 'https://non.test1.gov/a/b/c', + rollUpDate: 'Jan-21', + resourceCount: '2', + }, + ]); + }); + + it('resourceTopicUseFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { topicUseResult } = await resourceFlatData(scopes); + expect(topicUseResult).toBeDefined(); + + expect(topicUseResult).toStrictEqual([ + { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2' }, + { name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4' }, + { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3' }, + { name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2' }, + { name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2' }, + ]); + }); + + it('overviewFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { overView } = await resourceFlatData(scopes); + expect(overView).toBeDefined(); + const { + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources, + } = overView; + + // Number of Participants. + expect(numberOfParticipants).toStrictEqual([{ + participants: '44', + }]); + + // Number of Recipients. + expect(numberOfRecipients).toStrictEqual([{ + recipients: '1', + }]); + + // Percent of Reports with Resources. + expect(pctOfReportsWithResources).toStrictEqual([ + { + reportsWithResourcesCount: '4', + totalReportsCount: '5', + resourcesPct: '80.0000', + }, + ]); + + // Percent of ECLKC reports. + expect(pctOfECKLKCResources).toStrictEqual([ + { + eclkcCount: '2', + allCount: '3', + eclkcPct: '66.6667', + }, + ]); + }); +}); diff --git a/src/services/users.js b/src/services/users.js index 4b338babfb..3611b33c56 100644 --- a/src/services/users.js +++ b/src/services/users.js @@ -94,7 +94,6 @@ export async function userEmailIsVerifiedByUserId(userId) { /* Get Statistics by User */ export async function statisticsByUser(user, regions, readonly = false, reportIds = []) { - // Get days joined. const dateJoined = new Date(user.createdAt); const todaysDate = new Date(); From 925725f58b7065ac14ebd8c78369d7f401b8f0d4 Mon Sep 17 00:00:00 2001 From: nvms Date: Mon, 25 Mar 2024 14:44:04 -0400 Subject: [PATCH 07/75] deploy to sandbox --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 79c81d9aa3..23bc25001a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -277,7 +277,7 @@ parameters: default: "jp/2128/display-goal-creator-name" type: string sandbox_git_branch: # change to feature branch to test deployment - default: "kw-fix-fkey-error" + default: "jp/1185/validate-on-click" type: string prod_new_relic_app_id: default: "877570491" From 7fb8c1d42e04b28d97623f968b508ceec0772116 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 26 Mar 2024 12:05:05 -0400 Subject: [PATCH 08/75] handle all resource use clacs in sql --- src/lib/cache.ts | 6 +- src/services/dashboards/resource.js | 63 ++++++++++++++++---- src/services/dashboards/resourceFlat.test.js | 3 + 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index d62450df90..4c55a8ae8d 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -37,17 +37,18 @@ export default async function getCachedResponse( // we create a fake redis client because we don't want to fail the request if redis is down // or if we can't connect to it, or whatever else might go wrong - let redisClient = { + const redisClient = { connect: () => Promise.resolve(), get: (_k: string) => Promise.resolve(null), set: (_k: string, _r: string | null, _o: CacheOptions) => Promise.resolve(''), quit: () => Promise.resolve(), }; - let clientConnected = false; + const clientConnected = false; let response: string | null = null; try { + /* if (!ignoreCache) { redisClient = createClient({ url: redisUrl, @@ -59,6 +60,7 @@ export default async function getCachedResponse( response = await redisClient.get(key); clientConnected = true; } + */ } catch (err) { auditLogger.error('Error creating & connecting to redis client', { err }); } diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 0088337019..aa93874d58 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -450,8 +450,6 @@ export async function resourceFlatData(scopes) { JOIN ${createdResourcesTempTableName} arorr ON aror."resourceId" = arorr.id `; - - // await sequelize.query('SELECT * FROM projects', { raw: true }); const transaction = await sequelize.transaction(); // Create base tables. @@ -466,14 +464,50 @@ export async function resourceFlatData(scopes) { // Get resource use result. let resourceUseResult = sequelize.query( ` - SELECT - url, - "rollUpDate", - count(id) AS "resourceCount" - FROM ${createdFlatResourceTempTableName} tf - GROUP BY url, "rollUpDate" - ORDER BY "url", tf."rollUpDate" ASC - LIMIT 10; + WITH urlvals AS ( + SELECT + url, + "rollUpDate", + count(id) AS "resourceCount" + FROM ${createdFlatResourceTempTableName} tf + GROUP BY url, "rollUpDate" + ORDER BY "url", tf."rollUpDate" ASC), + distincturls AS ( + SELECT DISTINCT url AS url + FROM ${createdFlatResourceTempTableName} + ), + totals AS + ( + SELECT + url, + SUM("resourceCount") AS "totalCount" + FROM urlvals + GROUP BY url + ORDER BY SUM("resourceCount") DESC + LIMIT 10 + ), + series AS + ( + SELECT + generate_series( + date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), + date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), + interval '1 month' + )::date AS "date" + ) + SELECT + d.url, + to_char(s."date", 'Mon-YY') AS "rollUpDate", + coalesce(u."resourceCount", 0) AS "resourceCount", + t."totalCount" + FROM distincturls d + JOIN series s + ON 1=1 + JOIN totals t + ON d.url = t.url + LEFT JOIN urlvals u + ON d.url = u.url AND to_char(s."date", 'Mon-YY') = u."rollUpDate" + ORDER BY 1,2; `, { type: QueryTypes.SELECT, @@ -592,7 +626,8 @@ export async function resourceFlatData(scopes) { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, - pctOfECKLKCResources] = await Promise.all( + pctOfECKLKCResources, + ] = await Promise.all( [ resourceUseResult, topicUseResult, @@ -1889,3 +1924,9 @@ export async function resourceDashboard(scopes) { domainList: generateResourceDomainList(data, true), }; } + +export async function resourceDashboardFlat(scopes) { + // Get resources from SQL. + const data = await resourceFlatData(scopes); + return data; +} diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 86704e9ee8..1c97477d32 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -476,16 +476,19 @@ describe('Resources dashboard', () => { url: 'https://eclkc.ohs.acf.hhs.gov/test', rollUpDate: 'Jan-21', resourceCount: '2', + totalCount: '2', }, { url: 'https://eclkc.ohs.acf.hhs.gov/test2', rollUpDate: 'Jan-21', resourceCount: '3', + totalCount: '3', }, { url: 'https://non.test1.gov/a/b/c', rollUpDate: 'Jan-21', resourceCount: '2', + totalCount: '2', }, ]); }); From 15ff44e40cd8f97b09ddc40b38ce1f2405668831 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 26 Mar 2024 14:02:06 -0400 Subject: [PATCH 09/75] update topic use to be all sql --- src/services/dashboards/resource.js | 75 ++++++++++++++++---- src/services/dashboards/resourceFlat.test.js | 37 ++++++++-- 2 files changed, 90 insertions(+), 22 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index aa93874d58..dfd097434b 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -336,6 +336,7 @@ const switchToTopicCentric = (input) => { If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ export async function resourceFlatData(scopes) { + // console.time('overalltime'); // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -359,6 +360,8 @@ export async function resourceFlatData(scopes) { raw: true, }); + // console.log('\n\n\n----Report Count: ', reportIds.length, '\n\n\n'); + // Get total number of reports. const totalReportCount = reportIds.length; @@ -450,8 +453,10 @@ export async function resourceFlatData(scopes) { JOIN ${createdResourcesTempTableName} arorr ON aror."resourceId" = arorr.id `; + // console.log('\n\n\n-----sql: ', flatResourceSql, '\n\n\n'); const transaction = await sequelize.transaction(); + // console.time('maincreate'); // Create base tables. await sequelize.query( flatResourceSql, @@ -460,7 +465,8 @@ export async function resourceFlatData(scopes) { transaction, }, ); - + // console.timeEnd('maincreate'); + // console.time('resourceUseTime'); // Get resource use result. let resourceUseResult = sequelize.query( ` @@ -514,24 +520,62 @@ export async function resourceFlatData(scopes) { transaction, }, ); + // console.timeEnd('resourceUseTime'); + // console.time('topicUseTime'); // Get topic use result. - let topicUseResult = sequelize.query(` - SELECT - t.name, - f."rollUpDate", - count(f.id) AS "resourceCount" - FROM ${createdTopicsTempTableName} t - JOIN ${createdAroTopicsTempTableName} arot + let topicUseResult = sequelize.query( + ` + WITH topics AS ( + SELECT + t.name, + f."rollUpDate", + count(f.id) AS "resourceCount" + FROM ${createdTopicsTempTableName} t + JOIN ${createdAroTopicsTempTableName} arot ON t.id = arot."topicId" - JOIN ${createdFlatResourceTempTableName} f + JOIN ${createdFlatResourceTempTableName} f ON arot."activityReportId" = f.id - GROUP BY t.name, f."rollUpDate" - ORDER BY t.name, f."rollUpDate" ASC; - `, { - type: QueryTypes.SELECT, - transaction, - }); + GROUP BY t.name, f."rollUpDate" + ORDER BY t.name, f."rollUpDate" ASC + ), + totals AS + ( + SELECT + name, + SUM("resourceCount") AS "totalCount" + FROM topics + GROUP BY name + ORDER BY SUM("resourceCount") DESC + ), + series AS + ( + SELECT + generate_series( + date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), + date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), + interval '1 month' + )::date AS "date" + ) + SELECT + d.name, + to_char(s."date", 'Mon-YY') AS "rollUpDate", + coalesce(t."resourceCount", 0) AS "resourceCount", + tt."totalCount" + FROM ${createdTopicsTempTableName} d + JOIN series s + ON 1=1 + JOIN totals tt + ON d.name = tt.name + LEFT JOIN topics t + ON d.name = t.name AND to_char(s."date", 'Mon-YY') = t."rollUpDate" + ORDER BY 1,2;`, + { + type: QueryTypes.SELECT, + transaction, + }, + ); + // console.timeEnd('topicUseTime'); /* Overview */ // 1.) Participants @@ -644,6 +688,7 @@ export async function resourceFlatData(scopes) { const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, }; + // console.timeEnd('overalltime'); return { resourceUseResult, topicUseResult, numberOfParticipants, overView, }; diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 1c97477d32..ff70b53ffb 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -465,6 +465,15 @@ describe('Resources dashboard', () => { jest.clearAllMocks(); }); + /* + it('testAllReports', async () => { + const scopes = await filtersToScopes({}); + const { resourceUseResult } = await resourceFlatData(scopes); + // console.log('\n\n\n-----resourceUseResult:', resourceUseResult, '\n\n\n'); + expect(true).toBe(true); + }); + */ + it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); const { resourceUseResult } = await resourceFlatData(scopes); @@ -499,13 +508,27 @@ describe('Resources dashboard', () => { expect(topicUseResult).toBeDefined(); expect(topicUseResult).toStrictEqual([ - { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2' }, - { name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4' }, - { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3' }, - { name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2' }, - { name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2' }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + }, + { + name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4', totalCount: '4', + }, + { + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', + }, + { + name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', + }, + { + name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', + }, + { + name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + }, + { + name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + }, ]); }); From a9e8e0fd329a7956fab17c5a1a250986ace9bfd2 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 26 Mar 2024 15:22:50 -0400 Subject: [PATCH 10/75] create header table for now, unless we save a flat file in the future --- src/services/dashboards/resource.js | 46 +++++++++++++------- src/services/dashboards/resourceFlat.test.js | 15 +++++++ 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index dfd097434b..5a04beb9d8 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -371,6 +371,7 @@ export async function resourceFlatData(scopes) { const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; + const createdFlatResourceHeadersTempTableName = `Z_temp_flat_resources_headers__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. // Create raw sql to get flat table. @@ -451,11 +452,20 @@ export async function resourceFlatData(scopes) { JOIN ${createdAroResourcesTempTableName} aror ON ar.id = aror."activityReportId" JOIN ${createdResourcesTempTableName} arorr - ON aror."resourceId" = arorr.id + ON aror."resourceId" = arorr.id; + + -- 7.) Create date headers. + SELECT + generate_series( + date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), + date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), + interval '1 month' + )::date AS "date" + INTO TEMP ${createdFlatResourceHeadersTempTableName}; `; // console.log('\n\n\n-----sql: ', flatResourceSql, '\n\n\n'); const transaction = await sequelize.transaction(); - + // console.log('\n\n\n------AFter run sql'); // console.time('maincreate'); // Create base tables. await sequelize.query( @@ -473,10 +483,11 @@ export async function resourceFlatData(scopes) { WITH urlvals AS ( SELECT url, + title, "rollUpDate", count(id) AS "resourceCount" FROM ${createdFlatResourceTempTableName} tf - GROUP BY url, "rollUpDate" + GROUP BY url, title, "rollUpDate" ORDER BY "url", tf."rollUpDate" ASC), distincturls AS ( SELECT DISTINCT url AS url @@ -494,15 +505,11 @@ export async function resourceFlatData(scopes) { ), series AS ( - SELECT - generate_series( - date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), - date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), - interval '1 month' - )::date AS "date" + SELECT * FROM ${createdFlatResourceHeadersTempTableName} ) SELECT d.url, + u.title, to_char(s."date", 'Mon-YY') AS "rollUpDate", coalesce(u."resourceCount", 0) AS "resourceCount", t."totalCount" @@ -550,12 +557,7 @@ export async function resourceFlatData(scopes) { ), series AS ( - SELECT - generate_series( - date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), - date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), - interval '1 month' - )::date AS "date" + SELECT * FROM ${createdFlatResourceHeadersTempTableName} ) SELECT d.name, @@ -664,6 +666,16 @@ export async function resourceFlatData(scopes) { transaction, }); + // 5.) Date Headers table. + let dateHeaders = sequelize.query(` + SELECT + to_char("date", 'Mon-YY') AS "rollUpDate" + FROM ${createdFlatResourceHeadersTempTableName}; + `, { + type: QueryTypes.SELECT, + transaction, + }); + [ resourceUseResult, topicUseResult, @@ -671,6 +683,7 @@ export async function resourceFlatData(scopes) { numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, + dateHeaders, ] = await Promise.all( [ resourceUseResult, @@ -679,6 +692,7 @@ export async function resourceFlatData(scopes) { numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, + dateHeaders, ], ); @@ -690,7 +704,7 @@ export async function resourceFlatData(scopes) { }; // console.timeEnd('overalltime'); return { - resourceUseResult, topicUseResult, numberOfParticipants, overView, + resourceUseResult, topicUseResult, numberOfParticipants, overView, dateHeaders, }; } diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index ff70b53ffb..f1984aa1d1 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -484,18 +484,21 @@ describe('Resources dashboard', () => { { url: 'https://eclkc.ohs.acf.hhs.gov/test', rollUpDate: 'Jan-21', + title: null, resourceCount: '2', totalCount: '2', }, { url: 'https://eclkc.ohs.acf.hhs.gov/test2', rollUpDate: 'Jan-21', + title: null, resourceCount: '3', totalCount: '3', }, { url: 'https://non.test1.gov/a/b/c', rollUpDate: 'Jan-21', + title: null, resourceCount: '2', totalCount: '2', }, @@ -571,4 +574,16 @@ describe('Resources dashboard', () => { }, ]); }); + + it('resourceDateHeadersFlat', async () => { + const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); + const { dateHeaders } = await resourceFlatData(scopes); + expect(dateHeaders).toBeDefined(); + expect(dateHeaders.length).toBe(1); + expect(dateHeaders).toStrictEqual([ + { + rollUpDate: 'Jan-21', + }, + ]); + }); }); From 3757d89997b7fc85a9c5a0b53a1783b60b506005 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 26 Mar 2024 17:01:19 -0400 Subject: [PATCH 11/75] rollup topics and resource use --- src/services/dashboards/resource.js | 53 +++++++++- src/services/dashboards/resourceFlat.test.js | 102 ++++++++++++++++++- 2 files changed, 149 insertions(+), 6 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 5a04beb9d8..764aa2ff0b 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -336,7 +336,7 @@ const switchToTopicCentric = (input) => { If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ export async function resourceFlatData(scopes) { - // console.time('overalltime'); + console.time('overalltime'); // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -698,11 +698,11 @@ export async function resourceFlatData(scopes) { // Commit is required to run the query. transaction.commit(); - + console.timeEnd('overalltime'); const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, }; - // console.timeEnd('overalltime'); + return { resourceUseResult, topicUseResult, numberOfParticipants, overView, dateHeaders, }; @@ -1984,8 +1984,55 @@ export async function resourceDashboard(scopes) { }; } +export async function rollUpResourceUse(data) { + return data.resourceUseResults.reduce((accumulator, resource) => { + const exists = accumulator.find((r) => r.url === resource.url); + if (!exists) { + // Add a property with the resource's URL. + return [ + ...accumulator, + { + url: resource.url, + resources: [{ ...resource }], + }, + ]; + } + + // Add the resource to the accumulator. + exists.resources.push(resource); + return accumulator; + }, []); +} + +export async function rollUpTopicUse(data) { + return data.topicUseResult.reduce((accumulator, topic) => { + const exists = accumulator.find((r) => r.name === topic.name); + if (!exists) { + // Add a property with the resource's name. + return [ + ...accumulator, + { + name: topic.name, + topics: [{ ...topic }], + }, + ]; + } + + // Add the resource to the accumulator. + exists.topics.push(topic); + return accumulator; + }, []); +} + export async function resourceDashboardFlat(scopes) { // Get resources from SQL. const data = await resourceFlatData(scopes); + + // Roll up resource use data to each distinct url. + const rolledUpResourceUse = await rollUpResourceUse(data); + + // Roll up resource use data to each distinct url. + const rolledUpTopicUse = await rollUpTopicUse(data); + return data; } diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index f1984aa1d1..23111368ee 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -16,6 +16,8 @@ import db, { import filtersToScopes from '../../scopes'; import { resourceFlatData, + rollUpResourceUse, + rollUpTopicUse, } from './resource'; import { RESOURCE_DOMAIN } from '../../constants'; import { processActivityReportObjectiveForResourcesById } from '../resource'; @@ -465,14 +467,11 @@ describe('Resources dashboard', () => { jest.clearAllMocks(); }); - /* it('testAllReports', async () => { const scopes = await filtersToScopes({}); const { resourceUseResult } = await resourceFlatData(scopes); - // console.log('\n\n\n-----resourceUseResult:', resourceUseResult, '\n\n\n'); expect(true).toBe(true); }); - */ it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); @@ -586,4 +585,101 @@ describe('Resources dashboard', () => { }, ]); }); + + it('should roll up resource use results correctly', async () => { + const data = { + resourceUseResults: [ + { url: 'http://google.com', resourceCount: 1, rollUpDate: 'Jan-21' }, + { url: 'http://google.com', resourceCount: 2, rollUpDate: 'Feb-21' }, + { url: 'http://google.com', resourceCount: 3, rollUpDate: 'Mar-21' }, + { url: 'http://google.com', resourceCount: 4, rollUpDate: 'Apr-21' }, + { url: 'http://github.com', resourceCount: 1, rollUpDate: 'Jan-21' }, + { url: 'http://github.com', resourceCount: 2, rollUpDate: 'Feb-21' }, + { url: 'http://github.com', resourceCount: 3, rollUpDate: 'Mar-21' }, + { url: 'http://github.com', resourceCount: 4, rollUpDate: 'Apr-21' }, + { url: 'http://yahoo.com', resourceCount: 1, rollUpDate: 'Jan-21' }, + { url: 'http://yahoo.com', resourceCount: 2, rollUpDate: 'Feb-21' }, + { url: 'http://yahoo.com', resourceCount: 3, rollUpDate: 'Mar-21' }, + { url: 'http://yahoo.com', resourceCount: 4, rollUpDate: 'Apr-21' }, + ], + }; + + const result = await rollUpResourceUse(data); + + expect(result).toEqual([ + { + url: 'http://google.com', + resources: [ + { url: 'http://google.com', rollUpDate: 'Jan-21', resourceCount: 1 }, + { url: 'http://google.com', rollUpDate: 'Feb-21', resourceCount: 2 }, + { url: 'http://google.com', rollUpDate: 'Mar-21', resourceCount: 3 }, + { url: 'http://google.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + ], + }, + { + url: 'http://github.com', + resources: [ + { url: 'http://github.com', rollUpDate: 'Jan-21', resourceCount: 1 }, + { url: 'http://github.com', rollUpDate: 'Feb-21', resourceCount: 2 }, + { url: 'http://github.com', rollUpDate: 'Mar-21', resourceCount: 3 }, + { url: 'http://github.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + ], + }, + { + url: 'http://yahoo.com', + resources: [ + { url: 'http://yahoo.com', rollUpDate: 'Jan-21', resourceCount: 1 }, + { url: 'http://yahoo.com', rollUpDate: 'Feb-21', resourceCount: 2 }, + { url: 'http://yahoo.com', rollUpDate: 'Mar-21', resourceCount: 3 }, + { url: 'http://yahoo.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + ], + }, + ]); + }); + + it('should roll up topic use results correctly', async () => { + const data = { + topicUseResult: [ + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1', + }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2', + }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3', + }, + { + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1', + }, + { + name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2', + }, + { + name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3', + }, + ], + }; + + const result = await rollUpTopicUse(data); + + expect(result).toEqual([ + { + name: 'CLASS: Classroom Organization', + topics: [ + { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2' }, + { name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3' }, + ], + }, + { + name: 'ERSEA', + topics: [ + { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1' }, + { name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2' }, + { name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3' }, + ], + }, + ]); + }); }); From d5788adb0c8b8d92265a8b56021a1c03783bc8cd Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 26 Mar 2024 17:20:53 -0400 Subject: [PATCH 12/75] sort by date --- src/services/dashboards/resource.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 764aa2ff0b..a2da044812 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -511,6 +511,7 @@ export async function resourceFlatData(scopes) { d.url, u.title, to_char(s."date", 'Mon-YY') AS "rollUpDate", + s."date", coalesce(u."resourceCount", 0) AS "resourceCount", t."totalCount" FROM distincturls d @@ -520,7 +521,7 @@ export async function resourceFlatData(scopes) { ON d.url = t.url LEFT JOIN urlvals u ON d.url = u.url AND to_char(s."date", 'Mon-YY') = u."rollUpDate" - ORDER BY 1,2; + ORDER BY 1,4 ASC; `, { type: QueryTypes.SELECT, @@ -537,6 +538,7 @@ export async function resourceFlatData(scopes) { SELECT t.name, f."rollUpDate", + count(f.id) AS "resourceCount" FROM ${createdTopicsTempTableName} t JOIN ${createdAroTopicsTempTableName} arot @@ -562,6 +564,7 @@ export async function resourceFlatData(scopes) { SELECT d.name, to_char(s."date", 'Mon-YY') AS "rollUpDate", + s."date", coalesce(t."resourceCount", 0) AS "resourceCount", tt."totalCount" FROM ${createdTopicsTempTableName} d @@ -571,7 +574,7 @@ export async function resourceFlatData(scopes) { ON d.name = tt.name LEFT JOIN topics t ON d.name = t.name AND to_char(s."date", 'Mon-YY') = t."rollUpDate" - ORDER BY 1,2;`, + ORDER BY 1, 3 ASC;`, { type: QueryTypes.SELECT, transaction, From f2c4ea8b5d1816367618b76c542faf12be83db2f Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 27 Mar 2024 09:05:05 -0400 Subject: [PATCH 13/75] get test passing --- src/services/dashboards/resource.js | 4 ++-- src/services/dashboards/resourceFlat.test.js | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index a2da044812..679d5af5ab 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -336,7 +336,7 @@ const switchToTopicCentric = (input) => { If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ export async function resourceFlatData(scopes) { - console.time('overalltime'); + // console.time('overalltime'); // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -701,7 +701,7 @@ export async function resourceFlatData(scopes) { // Commit is required to run the query. transaction.commit(); - console.timeEnd('overalltime'); + // console.timeEnd('overalltime'); const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, }; diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 23111368ee..386c543030 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -467,11 +467,14 @@ describe('Resources dashboard', () => { jest.clearAllMocks(); }); + // eslint-disable-next-line jest/no-commented-out-tests + /* it('testAllReports', async () => { const scopes = await filtersToScopes({}); const { resourceUseResult } = await resourceFlatData(scopes); expect(true).toBe(true); }); + */ it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); From a747c16059d54fc94c07b8e5ea1e66596350821f Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 27 Mar 2024 10:56:04 -0400 Subject: [PATCH 14/75] for dev testing --- .circleci/config.yml | 2 +- frontend/src/fetchers/Resources.js | 9 ++ .../src/pages/ResourcesDashboard/index.js | 37 ++++++- src/routes/resources/handlers.js | 13 ++- src/routes/resources/index.js | 2 + src/services/dashboards/resource.js | 104 ++++++++++-------- src/services/dashboards/resourceFlat.test.js | 2 +- 7 files changed, 116 insertions(+), 53 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3a262be0fd..0b08792e1f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -274,7 +274,7 @@ parameters: type: string dev_git_branch: # change to feature branch to test deployment description: "Name of github branch that will deploy to dev" - default: "mb/TTAHUB-2501/front-end-goal-name-filter" + default: "al/ttahub-2570/flat-resource-sql" type: string sandbox_git_branch: # change to feature branch to test deployment default: "kw-fix-fkey-error" diff --git a/frontend/src/fetchers/Resources.js b/frontend/src/fetchers/Resources.js index ea9b21fe8f..22c186bbf8 100644 --- a/frontend/src/fetchers/Resources.js +++ b/frontend/src/fetchers/Resources.js @@ -11,6 +11,15 @@ export const fetchResourceData = async (query) => { }; }; +export const fetchFlatResourceData = async (query) => { + const res = await get(join('/', 'api', 'resources', 'flat', `?${query}`)); + const data = await res.json(); + + return { + ...data, + }; +}; + export const fetchTopicResources = async (sortBy = 'updatedAt', sortDir = 'desc', offset = 0, limit = TOPICS_PER_PAGE, filters) => { const request = join('/', 'api', 'resources', 'topic-resources', `?sortBy=${sortBy}&sortDir=${sortDir}&offset=${offset}&limit=${limit}${filters ? `&${filters}` : ''}`); const res = await get(request); diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index df671e4487..bbd9baaeb0 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -1,3 +1,5 @@ +/* eslint-disable no-alert */ +/* eslint-disable no-console */ import React, { useContext, useMemo, @@ -9,7 +11,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import PropTypes from 'prop-types'; import { Helmet } from 'react-helmet'; -import { Grid, Alert } from '@trussworks/react-uswds'; +import { Grid, Alert, Button } from '@trussworks/react-uswds'; import useDeepCompareEffect from 'use-deep-compare-effect'; import FilterPanel from '../../components/filter/FilterPanel'; import { allRegionsUserHasPermissionTo } from '../../permissions'; @@ -20,7 +22,7 @@ import ResourcesDashboardOverview from '../../widgets/ResourcesDashboardOverview import ResourceUse from '../../widgets/ResourceUse'; import { expandFilters, filtersToQueryString, formatDateRange } from '../../utils'; import './index.scss'; -import { fetchResourceData } from '../../fetchers/Resources'; +import { fetchResourceData, fetchFlatResourceData } from '../../fetchers/Resources'; import { downloadReports, getReportsViaIdPost, @@ -212,6 +214,36 @@ export default function ResourcesDashboard() { filtersToApply, ]); + const callFlatResources = async () => { + try { + setIsLoading(true); + const filterQuery = filtersToQueryString(filtersToApply); + // show an alert message with the time taken to fetch the data + + const timeBefore = new Date().getTime(); + const data = await fetchFlatResourceData( + filterQuery, + ); + const timeAfter = new Date().getTime(); + const timeTaken = timeAfter - timeBefore; + alert(`Time taken to fetch data: ${timeTaken} ms | ${timeTaken / 1000} seconds (see console for data)`); + + const { + overview, rolledUpResourceUse, rolledUpTopicUse, dateHeaders, + } = data; + console.log('overview:', overview); + console.log('rolledUpResourceUse:', rolledUpResourceUse); + console.log('rolledUpTopicUse:', rolledUpTopicUse); + console.log('dateHeaders:', dateHeaders); + setResourcesData(data); + updateError(''); + } catch (e) { + updateError('Unable to fetch FLAT resources'); + } finally { + setIsLoading(false); + } + }; + const handleDownloadReports = async (setIsDownloading, setDownloadError, url, buttonRef) => { try { setIsDownloading(true); @@ -291,6 +323,7 @@ export default function ResourcesDashboard() { )} + { + // console.log('\n\n\n------------------\n\n\n', data.resourceUseResult, '\n\n\n------------------\n\n\n'); + return data.resourceUseResult.reduce((accumulator, resource) => { const exists = accumulator.find((r) => r.url === resource.url); if (!exists) { // Add a property with the resource's URL. @@ -2029,13 +2032,18 @@ export async function rollUpTopicUse(data) { export async function resourceDashboardFlat(scopes) { // Get resources from SQL. + // console.time('sqlonly'); const data = await resourceFlatData(scopes); + // console.timeEnd('sqlonly'); + // console.time('rolluponly'); // Roll up resource use data to each distinct url. const rolledUpResourceUse = await rollUpResourceUse(data); // Roll up resource use data to each distinct url. const rolledUpTopicUse = await rollUpTopicUse(data); - - return data; + // console.timeEnd('rolluponly'); + return { + overview: data.overView, rolledUpResourceUse, rolledUpTopicUse, dateHeaders: data.dateHeaders, + }; } diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 386c543030..6c1014f9d0 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -591,7 +591,7 @@ describe('Resources dashboard', () => { it('should roll up resource use results correctly', async () => { const data = { - resourceUseResults: [ + resourceUseResult: [ { url: 'http://google.com', resourceCount: 1, rollUpDate: 'Jan-21' }, { url: 'http://google.com', resourceCount: 2, rollUpDate: 'Feb-21' }, { url: 'http://google.com', resourceCount: 3, rollUpDate: 'Mar-21' }, From b595d6203f22790555f68e34e9a409b3d0604771 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 27 Mar 2024 11:21:19 -0400 Subject: [PATCH 15/75] comment and fix tests --- src/lib/cache.test.js | 3 +++ src/services/dashboards/resourceFlat.test.js | 17 ++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/lib/cache.test.js b/src/lib/cache.test.js index ac720d3e2c..8e17c93edb 100644 --- a/src/lib/cache.test.js +++ b/src/lib/cache.test.js @@ -1,3 +1,4 @@ +/* eslint-disable jest/no-commented-out-tests */ import { createClient } from 'redis'; import getCachedResponse from './cache'; @@ -21,6 +22,7 @@ describe('getCachedResponse', () => { process.env = ORIGINAL_ENV; // restore original env }); + /* it('returns the cached response', async () => { const callback = jest.fn(() => 'new value'); createClient.mockImplementation(() => ({ @@ -32,6 +34,7 @@ describe('getCachedResponse', () => { const response = await getCachedResponse('key', callback); expect(response).toEqual('value'); }); + */ it('calls the callback when there is no cached response', async () => { createClient.mockImplementation(() => ({ diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 6c1014f9d0..3c90e5dad9 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -484,6 +484,7 @@ describe('Resources dashboard', () => { expect(resourceUseResult).toStrictEqual([ { + date: '2021-01-01', url: 'https://eclkc.ohs.acf.hhs.gov/test', rollUpDate: 'Jan-21', title: null, @@ -491,6 +492,7 @@ describe('Resources dashboard', () => { totalCount: '2', }, { + date: '2021-01-01', url: 'https://eclkc.ohs.acf.hhs.gov/test2', rollUpDate: 'Jan-21', title: null, @@ -498,6 +500,7 @@ describe('Resources dashboard', () => { totalCount: '3', }, { + date: '2021-01-01', url: 'https://non.test1.gov/a/b/c', rollUpDate: 'Jan-21', title: null, @@ -514,25 +517,25 @@ describe('Resources dashboard', () => { expect(topicUseResult).toStrictEqual([ { - name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', }, { - name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4', totalCount: '4', + name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4', totalCount: '4', date: '2021-01-01', }, { - name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', date: '2021-01-01', }, { - name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', + name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', }, { - name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', + name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', }, { - name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', }, { - name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', + name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', }, ]); }); From 5bb81c46fb6f5dcb8f538e684dfea5bfed558f56 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 28 Mar 2024 09:25:12 -0400 Subject: [PATCH 16/75] hook up cache to flat resource data set --- src/lib/cache.ts | 12 +++++------- src/routes/resources/handlers.js | 17 ++++++++++++----- src/routes/resources/index.js | 4 ++-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 4c55a8ae8d..0f27c3695e 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -9,14 +9,14 @@ interface CacheOptions { /** * * @param {string} key the key to use for the cache - * @param {function} reponseCallback will be called if the cache is empty (must return a string) + * @param {function} responseCallback will be called if the cache is empty (must return a string) * @param {function} outputCallback will be called to format the output, defaults to a passthrough * @param options see the interface above, defaults to 10 minutes * @returns Promise, the cached response or null if there was an error */ export default async function getCachedResponse( key: string, - reponseCallback: () => Promise, + responseCallback: () => Promise, outputCallback: ((foo: string) => string) | JSON['parse'] = (foo: string) => foo, options: CacheOptions = { EX: 600, @@ -37,18 +37,17 @@ export default async function getCachedResponse( // we create a fake redis client because we don't want to fail the request if redis is down // or if we can't connect to it, or whatever else might go wrong - const redisClient = { + let redisClient = { connect: () => Promise.resolve(), get: (_k: string) => Promise.resolve(null), set: (_k: string, _r: string | null, _o: CacheOptions) => Promise.resolve(''), quit: () => Promise.resolve(), }; - const clientConnected = false; + let clientConnected = false; let response: string | null = null; try { - /* if (!ignoreCache) { redisClient = createClient({ url: redisUrl, @@ -60,14 +59,13 @@ export default async function getCachedResponse( response = await redisClient.get(key); clientConnected = true; } - */ } catch (err) { auditLogger.error('Error creating & connecting to redis client', { err }); } // if we do not have a response, we need to call the callback if (!response) { - response = await reponseCallback(); + response = await responseCallback(); // and then, if we have a response and we are connected to redis, we need to set the cache if (response && clientConnected) { try { diff --git a/src/routes/resources/handlers.js b/src/routes/resources/handlers.js index 55575b6fe0..2181c2dacf 100644 --- a/src/routes/resources/handlers.js +++ b/src/routes/resources/handlers.js @@ -30,13 +30,20 @@ export async function getResourcesDashboardData(req, res) { res.json(response); } -export async function getFlatResourcesDashboardData(req, res) { - // console.time('overallendpoint'); +export async function getFlatResourcesDataWithCache(req, res) { const userId = await currentUserId(req, res); const query = await setReadRegions(req.query, userId); + const key = `getFlatResourcesDashboardData?v=${RESOURCE_DATA_CACHE_VERSION}&${JSON.stringify(query)}`; + + const response = await getCachedResponse( + key, + async () => { + const scopes = await filtersToScopes(query); + const data = await resourceDashboardFlat(scopes); + return JSON.stringify(data); + }, + JSON.parse, + ); - const scopes = await filtersToScopes(query); - const response = await resourceDashboardFlat(scopes); - // console.timeEnd('overallendpoint'); res.json(response); } diff --git a/src/routes/resources/index.js b/src/routes/resources/index.js index cbf9e87778..d62c10172a 100644 --- a/src/routes/resources/index.js +++ b/src/routes/resources/index.js @@ -1,11 +1,11 @@ import express from 'express'; import { getResourcesDashboardData, - getFlatResourcesDashboardData, + getFlatResourcesDataWithCache, } from './handlers'; import transactionWrapper from '../transactionWrapper'; const router = express.Router(); router.get('/', transactionWrapper(getResourcesDashboardData)); -router.get('/flat', transactionWrapper(getFlatResourcesDashboardData)); +router.get('/flat', transactionWrapper(getFlatResourcesDataWithCache)); export default router; From 95d59e33112c329bade3130a9e56710f1d4f7fee Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 28 Mar 2024 10:36:08 -0400 Subject: [PATCH 17/75] clean up and add test --- src/routes/resources/handlers.test.js | 30 +++- src/services/dashboards/resource.js | 247 ++++++++++++++------------ 2 files changed, 162 insertions(+), 115 deletions(-) diff --git a/src/routes/resources/handlers.test.js b/src/routes/resources/handlers.test.js index 4289fce6ad..49b95cb98b 100644 --- a/src/routes/resources/handlers.test.js +++ b/src/routes/resources/handlers.test.js @@ -1,9 +1,10 @@ -import { getResourcesDashboardData } from './handlers'; -import { resourceDashboardPhase1 } from '../../services/dashboards/resource'; +import { getResourcesDashboardData, getFlatResourcesDataWithCache } from './handlers'; +import { resourceDashboardPhase1, resourceDashboardFlat } from '../../services/dashboards/resource'; import { getUserReadRegions } from '../../services/accessValidation'; jest.mock('../../services/dashboards/resource', () => ({ resourceDashboardPhase1: jest.fn(), + resourceDashboardFlat: jest.fn(), })); jest.mock('../../services/accessValidation'); @@ -32,4 +33,29 @@ describe('Resources handler', () => { expect(res.json).toHaveBeenCalledWith(resourcesData); }); }); + describe('getFlatResourcesDataWithCache', () => { + it('should return all dashboard data', async () => { + const responseData = { + overview: {}, + rolledUpResourceUse: {}, + rolledUpTopicUse: {}, + dateHeaders: [], + }; + + resourceDashboardFlat.mockResolvedValue(responseData); + getUserReadRegions.mockResolvedValue([1]); + const req = { + session: { userId: 1 }, + query: { + sortBy: 'id', + direction: 'asc', + limit: 10, + offset: 0, + }, + }; + const res = { json: jest.fn() }; + await getFlatResourcesDataWithCache(req, res); + expect(res.json).toHaveBeenCalledWith(responseData); + }); + }); }); diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index f5cf3c8323..df41c76481 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -331,50 +331,7 @@ const switchToTopicCentric = (input) => { }); }; -/* - Create a flat table to calculate the resource data. Use temp tables to ONLY join to the rows we need. - If over time the amount of data increases and slows again we can cache the flat table a set frequency. -*/ -export async function resourceFlatData(scopes) { - // console.time('overalltime'); - // Date to retrieve report data from. - const reportCreatedAtDate = '2022-12-01'; - - // Get all ActivityReport ID's using SCOPES. - // We don't want to write custom filters. - - const reportIds = await ActivityReport.findAll({ - attributes: [ - 'id', - ], - where: { - [Op.and]: [ - scopes.activityReport, - { - calculatedStatus: REPORT_STATUSES.APPROVED, - startDate: { [Op.ne]: null }, - createdAt: { [Op.gt]: reportCreatedAtDate }, - }, - ], - }, - raw: true, - }); - - // console.log('\n\n\n----Report Count: ', reportIds.length, '\n\n\n'); - - // Get total number of reports. - const totalReportCount = reportIds.length; - - // Write raw sql to generate the flat resource data for the above reportIds. - const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; - const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; - const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; - const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; - const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; - const createdFlatResourceHeadersTempTableName = `Z_temp_flat_resources_headers__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. - const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. - - // Create raw sql to get flat table. +async function GenerateFlatTempTables(reportIds, tblNames) { const flatResourceSql = ` -- 1.) Create AR temp table. SELECT @@ -384,7 +341,7 @@ export async function resourceFlatData(scopes) { to_char("startDate", 'Mon-YY') AS "rollUpDate", "regionId", "calculatedStatus" - INTO TEMP ${createdArTempTableName} + INTO TEMP ${tblNames.createdArTempTableName} FROM "ActivityReports" ar WHERE ar."id" IN (${reportIds.map((r) => r.id).join(',')}); @@ -392,8 +349,8 @@ export async function resourceFlatData(scopes) { SELECT ar.id AS "activityReportId", aror."resourceId" - INTO TEMP ${createdAroResourcesTempTableName} - FROM ${createdArTempTableName} ar + INTO TEMP ${tblNames.createdAroResourcesTempTableName} + FROM ${tblNames.createdArTempTableName} ar JOIN "ActivityReportObjectives" aro ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveResources" aror @@ -407,11 +364,11 @@ export async function resourceFlatData(scopes) { domain, url, title - INTO TEMP ${createdResourcesTempTableName} + INTO TEMP ${tblNames.createdResourcesTempTableName} FROM "Resources" WHERE id IN ( SELECT DISTINCT "resourceId" - FROM ${createdAroResourcesTempTableName} + FROM ${tblNames.createdAroResourcesTempTableName} ); -- 4.) Create ARO Topics temp table. @@ -419,8 +376,8 @@ export async function resourceFlatData(scopes) { ar.id AS "activityReportId", arot."activityReportObjectiveId", -- We need to group by this incase of multiple aro's. arot."topicId" - INTO TEMP ${createdAroTopicsTempTableName} - FROM ${createdArTempTableName} ar + INTO TEMP ${tblNames.createdAroTopicsTempTableName} + FROM ${tblNames.createdArTempTableName} ar JOIN "ActivityReportObjectives" aro ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveTopics" arot @@ -431,11 +388,11 @@ export async function resourceFlatData(scopes) { SELECT id, name - INTO TEMP ${createdTopicsTempTableName} + INTO TEMP ${tblNames.createdTopicsTempTableName} FROM "Topics" WHERE id IN ( SELECT DISTINCT "topicId" - FROM ${createdAroTopicsTempTableName} + FROM ${tblNames.createdAroTopicsTempTableName} ); -- 6.) Create Flat Resource temp table. @@ -447,27 +404,25 @@ export async function resourceFlatData(scopes) { arorr.title, arorr.url, ar."numberOfParticipants" - INTO TEMP ${createdFlatResourceTempTableName} - FROM ${createdArTempTableName} ar - JOIN ${createdAroResourcesTempTableName} aror + INTO TEMP ${tblNames.createdFlatResourceTempTableName} + FROM ${tblNames.createdArTempTableName} ar + JOIN ${tblNames.createdAroResourcesTempTableName} aror ON ar.id = aror."activityReportId" - JOIN ${createdResourcesTempTableName} arorr + JOIN ${tblNames.createdResourcesTempTableName} arorr ON aror."resourceId" = arorr.id; -- 7.) Create date headers. SELECT generate_series( - date_trunc('month', (SELECT MIN("startDate") FROM ${createdFlatResourceTempTableName})), - date_trunc('month', (SELECT MAX("startDate") FROM ${createdFlatResourceTempTableName})), + date_trunc('month', (SELECT MIN("startDate") FROM ${tblNames.createdFlatResourceTempTableName})), + date_trunc('month', (SELECT MAX("startDate") FROM ${tblNames.createdFlatResourceTempTableName})), interval '1 month' )::date AS "date" - INTO TEMP ${createdFlatResourceHeadersTempTableName}; + INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; - // console.log('\n\n\n-----sql: ', flatResourceSql, '\n\n\n'); + const transaction = await sequelize.transaction(); - // console.log('\n\n\n------AFter run sql'); - // console.time('maincreate'); - // Create base tables. + // Execute the flat table sql. await sequelize.query( flatResourceSql, { @@ -475,22 +430,24 @@ export async function resourceFlatData(scopes) { transaction, }, ); - // console.timeEnd('maincreate'); - // console.time('resourceUseTime'); - // Get resource use result. - const resourceUseSQL = ` + return transaction; +} + +function getResourceUseSql(tblNames, transaction) { + return sequelize.query( + ` WITH urlvals AS ( SELECT url, title, "rollUpDate", count(id) AS "resourceCount" - FROM ${createdFlatResourceTempTableName} tf + FROM ${tblNames.createdFlatResourceTempTableName} tf GROUP BY url, title, "rollUpDate" ORDER BY "url", tf."rollUpDate" ASC), distincturls AS ( SELECT DISTINCT url AS url - FROM ${createdFlatResourceTempTableName} + FROM ${tblNames.createdFlatResourceTempTableName} ), totals AS ( @@ -504,7 +461,7 @@ export async function resourceFlatData(scopes) { ), series AS ( - SELECT * FROM ${createdFlatResourceHeadersTempTableName} + SELECT * FROM ${tblNames.createdFlatResourceHeadersTempTableName} ) SELECT d.url, @@ -521,20 +478,16 @@ export async function resourceFlatData(scopes) { LEFT JOIN urlvals u ON d.url = u.url AND to_char(s."date", 'Mon-YY') = u."rollUpDate" ORDER BY 1,4 ASC; - `; - // console.log('\n\n\n-----resourceUseSQL: ', resourceUseSQL, '\n\n\n'); - let resourceUseResult = sequelize.query( - resourceUseSQL, + `, { type: QueryTypes.SELECT, transaction, }, ); - // console.timeEnd('resourceUseTime'); +} - // console.time('topicUseTime'); - // Get topic use result. - let topicUseResult = sequelize.query( +function getTopicsUseSql(tblNames, transaction) { + return sequelize.query( ` WITH topics AS ( SELECT @@ -542,10 +495,10 @@ export async function resourceFlatData(scopes) { f."rollUpDate", count(f.id) AS "resourceCount" - FROM ${createdTopicsTempTableName} t - JOIN ${createdAroTopicsTempTableName} arot + FROM ${tblNames.createdTopicsTempTableName} t + JOIN ${tblNames.createdAroTopicsTempTableName} arot ON t.id = arot."topicId" - JOIN ${createdFlatResourceTempTableName} f + JOIN ${tblNames.createdFlatResourceTempTableName} f ON arot."activityReportId" = f.id GROUP BY t.name, f."rollUpDate" ORDER BY t.name, f."rollUpDate" ASC @@ -561,7 +514,7 @@ export async function resourceFlatData(scopes) { ), series AS ( - SELECT * FROM ${createdFlatResourceHeadersTempTableName} + SELECT * FROM ${tblNames.createdFlatResourceHeadersTempTableName} ) SELECT d.name, @@ -569,7 +522,7 @@ export async function resourceFlatData(scopes) { s."date", coalesce(t."resourceCount", 0) AS "resourceCount", tt."totalCount" - FROM ${createdTopicsTempTableName} d + FROM ${tblNames.createdTopicsTempTableName} d JOIN series s ON 1=1 JOIN totals tt @@ -582,16 +535,16 @@ export async function resourceFlatData(scopes) { transaction, }, ); - // console.timeEnd('topicUseTime'); +} - /* Overview */ - // 1.) Participants - let numberOfParticipants = sequelize.query(` +function getOverview(tblNames, totalReportCount, transaction) { + // - Number of Participants - + const numberOfParticipants = sequelize.query(` WITH ar_participants AS ( SELECT id, "numberOfParticipants" - FROM ${createdFlatResourceTempTableName} f + FROM ${tblNames.createdFlatResourceTempTableName} f GROUP BY id, "numberOfParticipants" ) SELECT @@ -602,12 +555,12 @@ export async function resourceFlatData(scopes) { transaction, }); - // 2.) Recipients. - let numberOfRecipients = sequelize.query(` + // - Number of Recipients - + const numberOfRecipients = sequelize.query(` WITH ars AS ( SELECT DISTINCT id - FROM ${createdFlatResourceTempTableName} f + FROM ${tblNames.createdFlatResourceTempTableName} f ), recipients AS ( SELECT DISTINCT r.id @@ -627,8 +580,8 @@ export async function resourceFlatData(scopes) { transaction, }); - // 3.) Reports with Resources. - let pctOfReportsWithResources = sequelize.query(` + // - Reports with Resources Pct - + const pctOfReportsWithResources = sequelize.query(` SELECT count(DISTINCT "activityReportId")::decimal AS "reportsWithResourcesCount", ${totalReportCount}::decimal AS "totalReportsCount", @@ -637,23 +590,23 @@ export async function resourceFlatData(scopes) { ELSE (round(count(DISTINCT "activityReportId")::decimal / ${totalReportCount}::decimal, 4) * 100)::decimal END AS "resourcesPct" - FROM ${createdAroResourcesTempTableName}; + FROM ${tblNames.createdAroResourcesTempTableName}; `, { type: QueryTypes.SELECT, transaction, }); - // 4.) ECKLKC resource percentage. - let pctOfECKLKCResources = sequelize.query(` + // - Number of Reports with ECLKC Resources Pct - + const pctOfECKLKCResources = sequelize.query(` WITH eclkc AS ( SELECT COUNT(DISTINCT url) AS "eclkcCount" - FROM ${createdFlatResourceTempTableName} + FROM ${tblNames.createdFlatResourceTempTableName} WHERE url ilike '%eclkc.ohs.acf.hhs.gov%' ), allres AS ( SELECT COUNT(DISTINCT url) AS "allCount" - FROM ${createdFlatResourceTempTableName} + FROM ${tblNames.createdFlatResourceTempTableName} ) SELECT e."eclkcCount", @@ -672,15 +625,91 @@ export async function resourceFlatData(scopes) { }); // 5.) Date Headers table. - let dateHeaders = sequelize.query(` + const dateHeaders = sequelize.query(` SELECT to_char("date", 'Mon-YY') AS "rollUpDate" - FROM ${createdFlatResourceHeadersTempTableName}; + FROM ${tblNames.createdFlatResourceHeadersTempTableName}; `, { type: QueryTypes.SELECT, transaction, }); + return { + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources, + dateHeaders, + }; +} + +/* + Create a flat table to calculate the resource data. Use temp tables to ONLY join to the rows we need. + If over time the amount of data increases and slows again we can cache the flat table a set frequency. +*/ +export async function resourceFlatData(scopes) { + // Date to retrieve report data from. + const reportCreatedAtDate = '2022-12-01'; + // Get all ActivityReport ID's using SCOPES. + // We don't want to write custom filters. + const reportIds = await ActivityReport.findAll({ + attributes: [ + 'id', + ], + where: { + [Op.and]: [ + scopes.activityReport, + { + calculatedStatus: REPORT_STATUSES.APPROVED, + startDate: { [Op.ne]: null }, + createdAt: { [Op.gt]: reportCreatedAtDate }, + }, + ], + }, + raw: true, + }); + + // Get total number of reports. + const totalReportCount = reportIds.length; + + // Create temp table names. + const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; + const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; + const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; + const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; + const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; + const createdFlatResourceHeadersTempTableName = `Z_temp_flat_resources_headers__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. + const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. + + const tempTableNames = { + createdArTempTableName, + createdAroResourcesTempTableName, + createdResourcesTempTableName, + createdAroTopicsTempTableName, + createdTopicsTempTableName, + createdFlatResourceTempTableName, + createdFlatResourceHeadersTempTableName, + }; + + // 1. Generate the flat sql temp tables. + const transaction = await GenerateFlatTempTables(reportIds, tempTableNames); + + // 2. Get resource use sql. + let resourceUseResult = getResourceUseSql(tempTableNames, transaction); + + // 3. Get topic use sql. + let topicUseResult = getTopicsUseSql(tempTableNames, transaction); + + // 4. Get Overview sql. + let { + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources, + dateHeaders, + } = getOverview(tempTableNames, totalReportCount, transaction); + + // 5. Execute all the sql queries. [ resourceUseResult, topicUseResult, @@ -701,13 +730,15 @@ export async function resourceFlatData(scopes) { ], ); - // Commit is required to run the query. + // 6. Commit is required to run the query. transaction.commit(); - // console.timeEnd('overalltime'); + + // 7. Restructure Overview. const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, }; + // 8. Return the data. return { resourceUseResult, topicUseResult, overView, dateHeaders, }; @@ -1990,7 +2021,6 @@ export async function resourceDashboard(scopes) { } export async function rollUpResourceUse(data) { - // console.log('\n\n\n------------------\n\n\n', data.resourceUseResult, '\n\n\n------------------\n\n\n'); return data.resourceUseResult.reduce((accumulator, resource) => { const exists = accumulator.find((r) => r.url === resource.url); if (!exists) { @@ -2031,18 +2061,9 @@ export async function rollUpTopicUse(data) { } export async function resourceDashboardFlat(scopes) { - // Get resources from SQL. - // console.time('sqlonly'); const data = await resourceFlatData(scopes); - // console.timeEnd('sqlonly'); - - // console.time('rolluponly'); - // Roll up resource use data to each distinct url. const rolledUpResourceUse = await rollUpResourceUse(data); - - // Roll up resource use data to each distinct url. const rolledUpTopicUse = await rollUpTopicUse(data); - // console.timeEnd('rolluponly'); return { overview: data.overView, rolledUpResourceUse, rolledUpTopicUse, dateHeaders: data.dateHeaders, }; From ab3715f0b0f9ad0c89ea89c53b1ae5b5f1574e75 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 28 Mar 2024 12:27:20 -0400 Subject: [PATCH 18/75] rollback a change to the old resource data func --- src/services/dashboards/resource.js | 127 ++++++++++--------- src/services/dashboards/resourceFlat.test.js | 29 +++++ 2 files changed, 94 insertions(+), 62 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index df41c76481..02716014ab 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -420,6 +420,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { )::date AS "date" INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; + // console.log('\n\n\n-----FLAT SQL', flatResourceSql); const transaction = await sequelize.transaction(); // Execute the flat table sql. @@ -555,8 +556,7 @@ function getOverview(tblNames, totalReportCount, transaction) { transaction, }); - // - Number of Recipients - - const numberOfRecipients = sequelize.query(` + const numberOfRecipSql = ` WITH ars AS ( SELECT DISTINCT id @@ -568,14 +568,18 @@ function getOverview(tblNames, totalReportCount, transaction) { JOIN "ActivityRecipients" arr ON ar.id = arr."activityReportId" JOIN "Grants" g - ON arr."grantId" = g.id + ON arr."grantId" = g.id AND g."status" = 'Active' JOIN "Recipients" r ON g."recipientId" = r.id ) SELECT count(r.id) AS recipients FROM recipients r; - `, { + `; + + // console.log('\n\n\n-----numberOfRecipSql', numberOfRecipSql); + // - Number of Recipients - + const numberOfRecipients = sequelize.query(numberOfRecipSql, { type: QueryTypes.SELECT, transaction, }); @@ -757,72 +761,71 @@ export async function resourceData(scopes, skipResources = false, skipTopics = f viaObjectives: null, viaGoals: null, }; - dbData.allReports = await ActivityReport.findAll({ - attributes: [ - 'id', - 'numberOfParticipants', - 'topics', - 'startDate', - [sequelize.fn( - 'jsonb_agg', - sequelize.fn( - 'DISTINCT', + [ + dbData.allReports, + // dbData.viaReport, + // dbData.viaSpecialistNextSteps, + // dbData.viaRecipientNextSteps, + dbData.viaObjectives, + dbData.viaGoals, + ] = await Promise.all([ + await ActivityReport.findAll({ + attributes: [ + 'id', + 'numberOfParticipants', + 'topics', + 'startDate', + [sequelize.fn( + 'jsonb_agg', sequelize.fn( - 'jsonb_build_object', - sequelize.literal('\'grantId\''), - sequelize.literal('"activityRecipients->grant"."id"'), - sequelize.literal('\'recipientId\''), - sequelize.literal('"activityRecipients->grant"."recipientId"'), - sequelize.literal('\'otherEntityId\''), - sequelize.literal('"activityRecipients"."otherEntityId"'), + 'DISTINCT', + sequelize.fn( + 'jsonb_build_object', + sequelize.literal('\'grantId\''), + sequelize.literal('"activityRecipients->grant"."id"'), + sequelize.literal('\'recipientId\''), + sequelize.literal('"activityRecipients->grant"."recipientId"'), + sequelize.literal('\'otherEntityId\''), + sequelize.literal('"activityRecipients"."otherEntityId"'), + ), ), ), - ), - 'recipients'], - ], - group: [ - '"ActivityReport"."id"', - '"ActivityReport"."numberOfParticipants"', - '"ActivityReport"."topics"', - '"ActivityReport"."startDate"', - ], - where: { - [Op.and]: [ - scopes.activityReport, - { - calculatedStatus: REPORT_STATUSES.APPROVED, - startDate: { [Op.ne]: null }, - createdAt: { [Op.gt]: reportCreatedAtDate }, - }, + 'recipients'], ], - }, - include: [ - { - model: ActivityRecipient.scope(), - as: 'activityRecipients', - attributes: [], - required: true, - include: [ + group: [ + '"ActivityReport"."id"', + '"ActivityReport"."numberOfParticipants"', + '"ActivityReport"."topics"', + '"ActivityReport"."startDate"', + ], + where: { + [Op.and]: [ + scopes.activityReport, { - model: Grant.scope(), - as: 'grant', - attributes: [], - required: false, + calculatedStatus: REPORT_STATUSES.APPROVED, + startDate: { [Op.ne]: null }, + createdAt: { [Op.gt]: reportCreatedAtDate }, }, ], }, - ], - raw: true, - }); - - [ - // dbData.allReports, - // dbData.viaReport, - // dbData.viaSpecialistNextSteps, - // dbData.viaRecipientNextSteps, - dbData.viaObjectives, - dbData.viaGoals, - ] = await Promise.all([ + include: [ + { + model: ActivityRecipient.scope(), + as: 'activityRecipients', + attributes: [], + required: true, + include: [ + { + model: Grant.scope(), + as: 'grant', + attributes: [], + required: false, + }, + ], + }, + ], + raw: true, + }), /* await ActivityReport.findAll({ attributes: [ diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 3c90e5dad9..c6ec239628 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -1,3 +1,5 @@ +/* eslint-disable jest/no-commented-out-tests */ +/* eslint-disable max-len */ import { REPORT_STATUSES } from '@ttahub/common'; import db, { ActivityReport, @@ -540,6 +542,33 @@ describe('Resources dashboard', () => { ]); }); + /* + it('testlocal', async () => { + const scopes = await filtersToScopes({ 'region.in': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'startDate.win': '2022/07/01-2022/07/31' }); + const { overView } = await resourceFlatData(scopes); + expect(overView).toBeDefined(); + const { + numberOfParticipants, + numberOfRecipients, + // pctOfReportsWithResources, + // pctOfECKLKCResources, + } = overView; + + console.log('\n\n\n---Result Recip:', numberOfRecipients); + console.log('\n\n\n---Result Particip:', numberOfParticipants); + + // Number of Recipients. + expect(numberOfRecipients).toStrictEqual([{ + recipients: '5', + }]); + + // Number of Participants. + expect(numberOfParticipants).toStrictEqual([{ + participants: '32', + }]); + }); + */ + it('overviewFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); const { overView } = await resourceFlatData(scopes); From b53bcc51293f6be1bfea30beb2ea264093eb83bb Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Fri, 29 Mar 2024 10:13:36 -0400 Subject: [PATCH 19/75] update sorting --- src/services/dashboards/resource.js | 16 +- src/services/dashboards/resourceFlat.test.js | 147 ++++++++++++++----- 2 files changed, 123 insertions(+), 40 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 02716014ab..6df0859aec 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -2024,7 +2024,7 @@ export async function resourceDashboard(scopes) { } export async function rollUpResourceUse(data) { - return data.resourceUseResult.reduce((accumulator, resource) => { + const rolledUpResourceUse = data.resourceUseResult.reduce((accumulator, resource) => { const exists = accumulator.find((r) => r.url === resource.url); if (!exists) { // Add a property with the resource's URL. @@ -2032,6 +2032,9 @@ export async function rollUpResourceUse(data) { ...accumulator, { url: resource.url, + title: resource.title, + sortBy: resource.title || resource.url, + total: resource.totalCount, resources: [{ ...resource }], }, ]; @@ -2041,10 +2044,14 @@ export async function rollUpResourceUse(data) { exists.resources.push(resource); return accumulator; }, []); + + // Sort by total and name or url. + rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.sortBy.localeCompare(r2.sortBy)); + return rolledUpResourceUse; } export async function rollUpTopicUse(data) { - return data.topicUseResult.reduce((accumulator, topic) => { + const rolledUpTopicUse = data.topicUseResult.reduce((accumulator, topic) => { const exists = accumulator.find((r) => r.name === topic.name); if (!exists) { // Add a property with the resource's name. @@ -2052,6 +2059,7 @@ export async function rollUpTopicUse(data) { ...accumulator, { name: topic.name, + total: topic.totalCount, topics: [{ ...topic }], }, ]; @@ -2061,6 +2069,10 @@ export async function rollUpTopicUse(data) { exists.topics.push(topic); return accumulator; }, []); + + // Sort by total then topic name. + rolledUpTopicUse.sort((r1, r2) => r2.total - r1.total || r1.name.localeCompare(r2.name)); + return rolledUpTopicUse; } export async function resourceDashboardFlat(scopes) { diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index c6ec239628..7eec5f5088 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -624,18 +624,42 @@ describe('Resources dashboard', () => { it('should roll up resource use results correctly', async () => { const data = { resourceUseResult: [ - { url: 'http://google.com', resourceCount: 1, rollUpDate: 'Jan-21' }, - { url: 'http://google.com', resourceCount: 2, rollUpDate: 'Feb-21' }, - { url: 'http://google.com', resourceCount: 3, rollUpDate: 'Mar-21' }, - { url: 'http://google.com', resourceCount: 4, rollUpDate: 'Apr-21' }, - { url: 'http://github.com', resourceCount: 1, rollUpDate: 'Jan-21' }, - { url: 'http://github.com', resourceCount: 2, rollUpDate: 'Feb-21' }, - { url: 'http://github.com', resourceCount: 3, rollUpDate: 'Mar-21' }, - { url: 'http://github.com', resourceCount: 4, rollUpDate: 'Apr-21' }, - { url: 'http://yahoo.com', resourceCount: 1, rollUpDate: 'Jan-21' }, - { url: 'http://yahoo.com', resourceCount: 2, rollUpDate: 'Feb-21' }, - { url: 'http://yahoo.com', resourceCount: 3, rollUpDate: 'Mar-21' }, - { url: 'http://yahoo.com', resourceCount: 4, rollUpDate: 'Apr-21' }, + { + url: 'http://google.com', resourceCount: 1, rollUpDate: 'Jan-21', title: null, totalCount: 10, + }, + { + url: 'http://google.com', resourceCount: 2, rollUpDate: 'Feb-21', title: null, totalCount: 10, + }, + { + url: 'http://google.com', resourceCount: 3, rollUpDate: 'Mar-21', title: null, totalCount: 10, + }, + { + url: 'http://google.com', resourceCount: 4, rollUpDate: 'Apr-21', title: null, totalCount: 10, + }, + { + url: 'http://github.com', resourceCount: 1, rollUpDate: 'Jan-21', title: null, totalCount: 10, + }, + { + url: 'http://github.com', resourceCount: 2, rollUpDate: 'Feb-21', title: null, totalCount: 10, + }, + { + url: 'http://github.com', resourceCount: 3, rollUpDate: 'Mar-21', title: null, totalCount: 10, + }, + { + url: 'http://github.com', resourceCount: 4, rollUpDate: 'Apr-21', title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', resourceCount: 1, rollUpDate: 'Jan-21', title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', resourceCount: 2, rollUpDate: 'Feb-21', title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', resourceCount: 3, rollUpDate: 'Mar-21', title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', resourceCount: 4, rollUpDate: 'Apr-21', title: null, totalCount: 10, + }, ], }; @@ -643,30 +667,63 @@ describe('Resources dashboard', () => { expect(result).toEqual([ { - url: 'http://google.com', + url: 'http://github.com', + total: 10, + title: null, + sortBy: 'http://github.com', resources: [ - { url: 'http://google.com', rollUpDate: 'Jan-21', resourceCount: 1 }, - { url: 'http://google.com', rollUpDate: 'Feb-21', resourceCount: 2 }, - { url: 'http://google.com', rollUpDate: 'Mar-21', resourceCount: 3 }, - { url: 'http://google.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + { + url: 'http://github.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + }, + { + url: 'http://github.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + }, + { + url: 'http://github.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + }, + { + url: 'http://github.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + }, ], }, { - url: 'http://github.com', + url: 'http://google.com', + title: null, + sortBy: 'http://google.com', + total: 10, resources: [ - { url: 'http://github.com', rollUpDate: 'Jan-21', resourceCount: 1 }, - { url: 'http://github.com', rollUpDate: 'Feb-21', resourceCount: 2 }, - { url: 'http://github.com', rollUpDate: 'Mar-21', resourceCount: 3 }, - { url: 'http://github.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + { + url: 'http://google.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + }, + { + url: 'http://google.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + }, + { + url: 'http://google.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + }, + { + url: 'http://google.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + }, ], }, { url: 'http://yahoo.com', + total: 10, + title: null, + sortBy: 'http://yahoo.com', resources: [ - { url: 'http://yahoo.com', rollUpDate: 'Jan-21', resourceCount: 1 }, - { url: 'http://yahoo.com', rollUpDate: 'Feb-21', resourceCount: 2 }, - { url: 'http://yahoo.com', rollUpDate: 'Mar-21', resourceCount: 3 }, - { url: 'http://yahoo.com', rollUpDate: 'Apr-21', resourceCount: 4 }, + { + url: 'http://yahoo.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + }, + { + url: 'http://yahoo.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + }, ], }, ]); @@ -676,22 +733,22 @@ describe('Resources dashboard', () => { const data = { topicUseResult: [ { - name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1', + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', }, { - name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2', + name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', }, { - name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3', + name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', }, { - name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1', + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', }, { - name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2', + name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', }, { - name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3', + name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', }, ], }; @@ -701,18 +758,32 @@ describe('Resources dashboard', () => { expect(result).toEqual([ { name: 'CLASS: Classroom Organization', + total: '6', topics: [ - { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2' }, - { name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3' }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', + }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', + }, + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', + }, ], }, { name: 'ERSEA', + total: '6', topics: [ - { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1' }, - { name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2' }, - { name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3' }, + { + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', + }, + { + name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', + }, + { + name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', + }, ], }, ]); From 18e09431a3952ef310749305d5a0b4a12ddc48d4 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Fri, 29 Mar 2024 14:49:22 -0400 Subject: [PATCH 20/75] match sorting on resource use for now --- src/services/dashboards/resource.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 6df0859aec..068d4b74f0 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -454,10 +454,13 @@ function getResourceUseSql(tblNames, transaction) { ( SELECT url, + title, SUM("resourceCount") AS "totalCount" FROM urlvals - GROUP BY url - ORDER BY SUM("resourceCount") DESC + GROUP BY url, title + ORDER BY SUM("resourceCount") DESC, + -- coalesce(title, url) ASC + url ASC LIMIT 10 ), series AS @@ -1914,7 +1917,7 @@ Expected JSON: title: 'Jan-22', value: '14', }, - { + {79 title: 'Feb-22', value: '20', }, @@ -2046,7 +2049,8 @@ export async function rollUpResourceUse(data) { }, []); // Sort by total and name or url. - rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.sortBy.localeCompare(r2.sortBy)); + // rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.sortBy.localeCompare(r2.sortBy)); + rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.url.localeCompare(r2.url)); return rolledUpResourceUse; } From e530e719e260594d4a84f27dae995a9166f9174e Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Fri, 29 Mar 2024 16:44:56 -0400 Subject: [PATCH 21/75] fix topics cound per AR --- src/services/dashboards/resource.js | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 068d4b74f0..7d47ac5f3e 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -491,28 +491,36 @@ function getResourceUseSql(tblNames, transaction) { } function getTopicsUseSql(tblNames, transaction) { - return sequelize.query( - ` + const topicUseSql = ` WITH topics AS ( SELECT + f.id, t.name, f."rollUpDate", - - count(f.id) AS "resourceCount" + count(DISTINCT f.url) AS "resourceCount" -- Only count each resource once per ar and topic. FROM ${tblNames.createdTopicsTempTableName} t JOIN ${tblNames.createdAroTopicsTempTableName} arot ON t.id = arot."topicId" JOIN ${tblNames.createdFlatResourceTempTableName} f ON arot."activityReportId" = f.id - GROUP BY t.name, f."rollUpDate" + GROUP BY f.id, t.name, f."rollUpDate" ORDER BY t.name, f."rollUpDate" ASC ), + topicsperdate AS + ( + SELECT + "name", + "rollUpDate", + SUM("resourceCount") AS "resourceCount" + FROM topics + GROUP BY "name", "rollUpDate" + ), totals AS ( SELECT name, SUM("resourceCount") AS "totalCount" - FROM topics + FROM topicsperdate GROUP BY name ORDER BY SUM("resourceCount") DESC ), @@ -531,9 +539,12 @@ function getTopicsUseSql(tblNames, transaction) { ON 1=1 JOIN totals tt ON d.name = tt.name - LEFT JOIN topics t + LEFT JOIN topicsperdate t ON d.name = t.name AND to_char(s."date", 'Mon-YY') = t."rollUpDate" - ORDER BY 1, 3 ASC;`, + ORDER BY 1, 3 ASC;`; + // console.log('\n\n\n--Topic use sql', topicUseSql); + return sequelize.query( + topicUseSql, { type: QueryTypes.SELECT, transaction, @@ -679,6 +690,8 @@ export async function resourceFlatData(scopes) { // Get total number of reports. const totalReportCount = reportIds.length; + // console.log('\n\n\n-----reportIds flat', reportIds); + // Create temp table names. const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; From f3e3786689a1d60b1f84756c9dc1be4c0b2a8f52 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 1 Apr 2024 11:11:44 -0400 Subject: [PATCH 22/75] clean up --- src/services/dashboards/resource.js | 30 +++++++------------- src/services/dashboards/resourceFlat.test.js | 29 +------------------ 2 files changed, 12 insertions(+), 47 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 7d47ac5f3e..5c95ff0157 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -420,7 +420,6 @@ async function GenerateFlatTempTables(reportIds, tblNames) { )::date AS "date" INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; - // console.log('\n\n\n-----FLAT SQL', flatResourceSql); const transaction = await sequelize.transaction(); // Execute the flat table sql. @@ -542,7 +541,6 @@ function getTopicsUseSql(tblNames, transaction) { LEFT JOIN topicsperdate t ON d.name = t.name AND to_char(s."date", 'Mon-YY') = t."rollUpDate" ORDER BY 1, 3 ASC;`; - // console.log('\n\n\n--Topic use sql', topicUseSql); return sequelize.query( topicUseSql, { @@ -591,7 +589,6 @@ function getOverview(tblNames, totalReportCount, transaction) { FROM recipients r; `; - // console.log('\n\n\n-----numberOfRecipSql', numberOfRecipSql); // - Number of Recipients - const numberOfRecipients = sequelize.query(numberOfRecipSql, { type: QueryTypes.SELECT, @@ -668,8 +665,7 @@ export async function resourceFlatData(scopes) { // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; - // Get all ActivityReport ID's using SCOPES. - // We don't want to write custom filters. + // 1.) Get report ids using the scopes. const reportIds = await ActivityReport.findAll({ attributes: [ 'id', @@ -689,10 +685,7 @@ export async function resourceFlatData(scopes) { // Get total number of reports. const totalReportCount = reportIds.length; - - // console.log('\n\n\n-----reportIds flat', reportIds); - - // Create temp table names. + // 2.) Create temp table names. const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; @@ -711,16 +704,18 @@ export async function resourceFlatData(scopes) { createdFlatResourceHeadersTempTableName, }; - // 1. Generate the flat sql temp tables. + // 3. Generate the base temp tables (used for all calcs). const transaction = await GenerateFlatTempTables(reportIds, tempTableNames); - // 2. Get resource use sql. + // 4.) Calculate the resource data. + + // -- Resource Use -- let resourceUseResult = getResourceUseSql(tempTableNames, transaction); - // 3. Get topic use sql. + // -- Topic Use -- let topicUseResult = getTopicsUseSql(tempTableNames, transaction); - // 4. Get Overview sql. + // -- Overview -- let { numberOfParticipants, numberOfRecipients, @@ -729,7 +724,7 @@ export async function resourceFlatData(scopes) { dateHeaders, } = getOverview(tempTableNames, totalReportCount, transaction); - // 5. Execute all the sql queries. + // -- Wait for all results -- [ resourceUseResult, topicUseResult, @@ -749,16 +744,13 @@ export async function resourceFlatData(scopes) { dateHeaders, ], ); - - // 6. Commit is required to run the query. transaction.commit(); - - // 7. Restructure Overview. + // 5.) Restructure Overview. const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, }; - // 8. Return the data. + // 6.) Return the data. return { resourceUseResult, topicUseResult, overView, dateHeaders, }; diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 7eec5f5088..88aca592cc 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -522,7 +522,7 @@ describe('Resources dashboard', () => { name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', }, { - name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '4', totalCount: '4', date: '2021-01-01', + name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', }, { name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', date: '2021-01-01', @@ -542,33 +542,6 @@ describe('Resources dashboard', () => { ]); }); - /* - it('testlocal', async () => { - const scopes = await filtersToScopes({ 'region.in': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], 'startDate.win': '2022/07/01-2022/07/31' }); - const { overView } = await resourceFlatData(scopes); - expect(overView).toBeDefined(); - const { - numberOfParticipants, - numberOfRecipients, - // pctOfReportsWithResources, - // pctOfECKLKCResources, - } = overView; - - console.log('\n\n\n---Result Recip:', numberOfRecipients); - console.log('\n\n\n---Result Particip:', numberOfParticipants); - - // Number of Recipients. - expect(numberOfRecipients).toStrictEqual([{ - recipients: '5', - }]); - - // Number of Participants. - expect(numberOfParticipants).toStrictEqual([{ - participants: '32', - }]); - }); - */ - it('overviewFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); const { overView } = await resourceFlatData(scopes); From 8e9ca951e81a8d7144abd9539f0fe23660c61b0e Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 1 Apr 2024 15:23:43 -0400 Subject: [PATCH 23/75] restructure the new flat dataset to match what we currently expect on the FE --- .../src/pages/ResourcesDashboard/index.js | 2 + src/services/dashboards/resource.js | 58 ++++++++- src/services/dashboards/resourceFlat.test.js | 113 +++++++++++++----- 3 files changed, 136 insertions(+), 37 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index bbd9baaeb0..9c4cd6cf38 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -200,6 +200,7 @@ export default function ResourcesDashboard() { const data = await fetchResourceData( filterQuery, ); + console.log('\n\n\n----- All Data: ', data); setResourcesData(data); updateError(''); } catch (e) { @@ -224,6 +225,7 @@ export default function ResourcesDashboard() { const data = await fetchFlatResourceData( filterQuery, ); + console.log('flat data: ', data); const timeAfter = new Date().getTime(); const timeTaken = timeAfter - timeBefore; alert(`Time taken to fetch data: ${timeTaken} ms | ${timeTaken / 1000} seconds (see console for data)`); diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 5c95ff0157..e342c7209e 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -745,6 +745,7 @@ export async function resourceFlatData(scopes) { ], ); transaction.commit(); + // 5.) Restructure Overview. const overView = { numberOfParticipants, numberOfRecipients, pctOfReportsWithResources, pctOfECKLKCResources, @@ -2039,20 +2040,27 @@ export async function rollUpResourceUse(data) { return [ ...accumulator, { + heading: resource.url, url: resource.url, title: resource.title, sortBy: resource.title || resource.url, total: resource.totalCount, - resources: [{ ...resource }], + isUrl: true, + data: [{ title: resource.rollUpDate, value: resource.resourceCount }], }, ]; } // Add the resource to the accumulator. - exists.resources.push(resource); + exists.data.push({ title: resource.rollUpDate, value: resource.resourceCount }); return accumulator; }, []); + // Loop through the rolled up resources and add a total. + rolledUpResourceUse.forEach((resource) => { + resource.data.push({ title: 'Total', value: resource.total }); + }); + // Sort by total and name or url. // rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.sortBy.localeCompare(r2.sortBy)); rolledUpResourceUse.sort((r1, r2) => r2.total - r1.total || r1.url.localeCompare(r2.url)); @@ -2067,28 +2075,68 @@ export async function rollUpTopicUse(data) { return [ ...accumulator, { + heading: topic.name, name: topic.name, total: topic.totalCount, - topics: [{ ...topic }], + isUrl: false, + data: [{ title: topic.rollUpDate, value: topic.resourceCount }], }, ]; } // Add the resource to the accumulator. - exists.topics.push(topic); + exists.data.push({ title: topic.rollUpDate, value: topic.resourceCount }); return accumulator; }, []); + // Loop through the rolled up resources and add a total. + rolledUpTopicUse.forEach((topic) => { + topic.data.push({ title: 'Total', value: topic.total }); + }); + // Sort by total then topic name. rolledUpTopicUse.sort((r1, r2) => r2.total - r1.total || r1.name.localeCompare(r2.name)); return rolledUpTopicUse; } +export function restructureOverview(data) { + return { + report: { + percentResources: data.overView.pctOfReportsWithResources[0].resourcesPct, + numResources: data.overView.pctOfReportsWithResources[0].reportsWithResourcesCount, + num: data.overView.pctOfReportsWithResources[0].totalReportsCount, + }, + participant: { + numParticipants: data.overView.numberOfParticipants[0].participants, + }, + recipient: { + numResources: data.overView.numberOfRecipients[0].recipients, + }, + resource: { + count: data.overView.pctOfECKLKCResources[0].eclkcCount, + total: data.overView.pctOfECKLKCResources[0].allCount, + percent: data.overView.pctOfECKLKCResources[0].eclkcPct, + }, + }; +} + export async function resourceDashboardFlat(scopes) { const data = await resourceFlatData(scopes); + + // Restructure overview. + const dashboardOverview = restructureOverview(data); + + // Roll up resources. const rolledUpResourceUse = await rollUpResourceUse(data); + + // Roll up topics. const rolledUpTopicUse = await rollUpTopicUse(data); + + // Date headers. + const dateHeadersArray = data.dateHeaders.map((date) => date.rollUpDate); return { - overview: data.overView, rolledUpResourceUse, rolledUpTopicUse, dateHeaders: data.dateHeaders, + resourcesDashboardOverview: dashboardOverview, + resourceUse: { headers: dateHeadersArray, resources: rolledUpResourceUse }, + topicUse: { headers: dateHeadersArray, topics: rolledUpTopicUse }, }; } diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 88aca592cc..2f2802fed2 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -20,6 +20,7 @@ import { resourceFlatData, rollUpResourceUse, rollUpTopicUse, + restructureOverview, } from './resource'; import { RESOURCE_DOMAIN } from '../../constants'; import { processActivityReportObjectiveForResourcesById } from '../resource'; @@ -469,15 +470,6 @@ describe('Resources dashboard', () => { jest.clearAllMocks(); }); - // eslint-disable-next-line jest/no-commented-out-tests - /* - it('testAllReports', async () => { - const scopes = await filtersToScopes({}); - const { resourceUseResult } = await resourceFlatData(scopes); - expect(true).toBe(true); - }); - */ - it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); const { resourceUseResult } = await resourceFlatData(scopes); @@ -640,62 +632,77 @@ describe('Resources dashboard', () => { expect(result).toEqual([ { + heading: 'http://github.com', url: 'http://github.com', total: 10, title: null, sortBy: 'http://github.com', - resources: [ + isUrl: true, + data: [ { - url: 'http://github.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + title: 'Jan-21', value: 1, }, { - url: 'http://github.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + title: 'Feb-21', value: 2, }, { - url: 'http://github.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + title: 'Mar-21', value: 3, }, { - url: 'http://github.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + title: 'Apr-21', value: 4, + }, + { + title: 'Total', value: 10, }, ], }, { + heading: 'http://google.com', url: 'http://google.com', title: null, sortBy: 'http://google.com', total: 10, - resources: [ + isUrl: true, + data: [ + { + title: 'Jan-21', value: 1, + }, { - url: 'http://google.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + title: 'Feb-21', value: 2, }, { - url: 'http://google.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + title: 'Mar-21', value: 3, }, { - url: 'http://google.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + title: 'Apr-21', value: 4, }, { - url: 'http://google.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + title: 'Total', value: 10, }, ], }, { + heading: 'http://yahoo.com', url: 'http://yahoo.com', total: 10, title: null, sortBy: 'http://yahoo.com', - resources: [ + isUrl: true, + data: [ + { + title: 'Jan-21', value: 1, + }, { - url: 'http://yahoo.com', rollUpDate: 'Jan-21', resourceCount: 1, title: null, totalCount: 10, + title: 'Feb-21', value: 2, }, { - url: 'http://yahoo.com', rollUpDate: 'Feb-21', resourceCount: 2, title: null, totalCount: 10, + title: 'Mar-21', value: 3, }, { - url: 'http://yahoo.com', rollUpDate: 'Mar-21', resourceCount: 3, title: null, totalCount: 10, + title: 'Apr-21', value: 4, }, { - url: 'http://yahoo.com', rollUpDate: 'Apr-21', resourceCount: 4, title: null, totalCount: 10, + title: 'Total', value: 10, }, ], }, @@ -730,35 +737,77 @@ describe('Resources dashboard', () => { expect(result).toEqual([ { + heading: 'CLASS: Classroom Organization', name: 'CLASS: Classroom Organization', total: '6', - topics: [ + isUrl: false, + data: [ { - name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', + title: 'Jan-21', value: '1', }, { - name: 'CLASS: Classroom Organization', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', + title: 'Feb-21', value: '2', }, { - name: 'CLASS: Classroom Organization', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', + title: 'Mar-21', value: '3', + }, + { + title: 'Total', value: '6', }, ], }, { + heading: 'ERSEA', name: 'ERSEA', total: '6', - topics: [ + isUrl: false, + data: [ + { + title: 'Jan-21', value: '1', + }, { - name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '6', + title: 'Feb-21', value: '2', }, { - name: 'ERSEA', rollUpDate: 'Feb-21', resourceCount: '2', totalCount: '6', + title: 'Mar-21', value: '3', }, { - name: 'ERSEA', rollUpDate: 'Mar-21', resourceCount: '3', totalCount: '6', + title: 'Total', value: '6', }, ], }, ]); }); + + it('verify overview restructures correctly', async () => { + const overviewData = { + overView: { + pctOfReportsWithResources: [{ resourcesPct: '80.0000', reportsWithResourcesCount: '4', totalReportsCount: '5' }], + numberOfParticipants: [{ participants: '44' }], + numberOfRecipients: [{ recipients: '1' }], + pctOfECKLKCResources: [{ eclkcCount: '2', allCount: '3', eclkcPct: '66.6667' }], + }, + }; + + const result = restructureOverview(overviewData); + + expect(result).toEqual({ + report: { + percentResources: '80.0000', + numResources: '4', + num: '5', + }, + participant: { + numParticipants: '44', + }, + recipient: { + numResources: '1', + }, + resource: { + count: '2', + total: '3', + percent: '66.6667', + }, + }); + }); }); From 4bb7dfbda8c40b30e10b49fcb5bb1b9b7893a441 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 1 Apr 2024 16:39:54 -0400 Subject: [PATCH 24/75] hook up new flat resources to FE --- .../src/pages/ResourcesDashboard/index.js | 51 ++++++------------- src/services/dashboards/resource.js | 25 +++++---- 2 files changed, 30 insertions(+), 46 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 9c4cd6cf38..04b202662a 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -11,7 +11,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import PropTypes from 'prop-types'; import { Helmet } from 'react-helmet'; -import { Grid, Alert, Button } from '@trussworks/react-uswds'; +import { Grid, Alert } from '@trussworks/react-uswds'; import useDeepCompareEffect from 'use-deep-compare-effect'; import FilterPanel from '../../components/filter/FilterPanel'; import { allRegionsUserHasPermissionTo } from '../../permissions'; @@ -22,6 +22,7 @@ import ResourcesDashboardOverview from '../../widgets/ResourcesDashboardOverview import ResourceUse from '../../widgets/ResourceUse'; import { expandFilters, filtersToQueryString, formatDateRange } from '../../utils'; import './index.scss'; +// eslint-disable-next-line no-unused-vars import { fetchResourceData, fetchFlatResourceData } from '../../fetchers/Resources'; import { downloadReports, @@ -197,10 +198,21 @@ export default function ResourcesDashboard() { // Filters passed also contains region. const filterQuery = filtersToQueryString(filtersToApply); try { - const data = await fetchResourceData( + /* + const oldData = await fetchResourceData( filterQuery, ); - console.log('\n\n\n----- All Data: ', data); + */ + const timeBefore = new Date().getTime(); + const data = await fetchFlatResourceData( + filterQuery, + ); + const timeAfter = new Date().getTime(); + const timeTaken = timeAfter - timeBefore; + alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); + + console.log('\n\n\n----- Flat Data: ', data); + // console.log('\n\n\n----- Old Data: ', oldData); setResourcesData(data); updateError(''); } catch (e) { @@ -215,37 +227,6 @@ export default function ResourcesDashboard() { filtersToApply, ]); - const callFlatResources = async () => { - try { - setIsLoading(true); - const filterQuery = filtersToQueryString(filtersToApply); - // show an alert message with the time taken to fetch the data - - const timeBefore = new Date().getTime(); - const data = await fetchFlatResourceData( - filterQuery, - ); - console.log('flat data: ', data); - const timeAfter = new Date().getTime(); - const timeTaken = timeAfter - timeBefore; - alert(`Time taken to fetch data: ${timeTaken} ms | ${timeTaken / 1000} seconds (see console for data)`); - - const { - overview, rolledUpResourceUse, rolledUpTopicUse, dateHeaders, - } = data; - console.log('overview:', overview); - console.log('rolledUpResourceUse:', rolledUpResourceUse); - console.log('rolledUpTopicUse:', rolledUpTopicUse); - console.log('dateHeaders:', dateHeaders); - setResourcesData(data); - updateError(''); - } catch (e) { - updateError('Unable to fetch FLAT resources'); - } finally { - setIsLoading(false); - } - }; - const handleDownloadReports = async (setIsDownloading, setDownloadError, url, buttonRef) => { try { setIsDownloading(true); @@ -325,7 +306,7 @@ export default function ResourcesDashboard() { )} - + date.rollUpDate); return { resourcesDashboardOverview: dashboardOverview, - resourceUse: { headers: dateHeadersArray, resources: rolledUpResourceUse }, + resourcesUse: { headers: dateHeadersArray, resources: rolledUpResourceUse }, topicUse: { headers: dateHeadersArray, topics: rolledUpTopicUse }, + reportIds: data.reportIds.map((r) => r.id), }; } From 3f7e673f85024f17b19e60c52d5b8d678385224e Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 1 Apr 2024 18:30:11 -0400 Subject: [PATCH 25/75] hook up FE --- frontend/src/pages/ResourcesDashboard/index.js | 18 ++++++++++-------- src/services/dashboards/resource.js | 16 ++++++++-------- src/services/dashboards/resourceFlat.test.js | 8 ++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 04b202662a..5ae307188d 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -198,21 +198,23 @@ export default function ResourcesDashboard() { // Filters passed also contains region. const filterQuery = filtersToQueryString(filtersToApply); try { - /* - const oldData = await fetchResourceData( + +/* const oldData = await fetchResourceData( filterQuery, ); */ - const timeBefore = new Date().getTime(); + + //const timeBefore = new Date().getTime(); const data = await fetchFlatResourceData( filterQuery, ); - const timeAfter = new Date().getTime(); - const timeTaken = timeAfter - timeBefore; - alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); + //const timeAfter = new Date().getTime(); + //const timeTaken = timeAfter - timeBefore; + //alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); + - console.log('\n\n\n----- Flat Data: ', data); - // console.log('\n\n\n----- Old Data: ', oldData); + // console.log('\n\n\n----- Flat Data: ', data); + //console.log('\n\n\n----- Old Data: ', oldData); setResourcesData(data); updateError(''); } catch (e) { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index bda7f9027c..97978c51e6 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -2103,20 +2103,20 @@ export async function rollUpTopicUse(data) { export function restructureOverview(data) { return { report: { - percentResources: data.overView.pctOfReportsWithResources[0].resourcesPct, - numResources: data.overView.pctOfReportsWithResources[0].reportsWithResourcesCount, - num: data.overView.pctOfReportsWithResources[0].totalReportsCount, + percentResources: `${formatNumber(data.overView.pctOfReportsWithResources[0].resourcesPct, 2)}%`, + numResources: formatNumber(data.overView.pctOfReportsWithResources[0].reportsWithResourcesCount), + num: formatNumber(data.overView.pctOfReportsWithResources[0].totalReportsCount), }, participant: { - numParticipants: data.overView.numberOfParticipants[0].participants, + numParticipants: formatNumber(data.overView.numberOfParticipants[0].participants), }, recipient: { - numResources: data.overView.numberOfRecipients[0].recipients, + numResources: formatNumber(data.overView.numberOfRecipients[0].recipients), }, resource: { - numEclkc: data.overView.pctOfECKLKCResources[0].eclkcCount, - num: data.overView.pctOfECKLKCResources[0].allCount, - percentEclkc: data.overView.pctOfECKLKCResources[0].eclkcPct, + numEclkc: formatNumber(data.overView.pctOfECKLKCResources[0].eclkcCount), + num: formatNumber(data.overView.pctOfECKLKCResources[0].allCount), + percentEclkc: `${formatNumber(data.overView.pctOfECKLKCResources[0].eclkcPct, 2)}%`, }, }; } diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 2f2802fed2..f36f6f139a 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -793,7 +793,7 @@ describe('Resources dashboard', () => { expect(result).toEqual({ report: { - percentResources: '80.0000', + percentResources: '80.00%', numResources: '4', num: '5', }, @@ -804,9 +804,9 @@ describe('Resources dashboard', () => { numResources: '1', }, resource: { - count: '2', - total: '3', - percent: '66.6667', + numEclkc: '2', + num: '3', + percentEclkc: '66.67%', }, }); }); From 610ab3eb7c66cdc75da552b68218e6ebdc02b350 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 1 Apr 2024 18:32:29 -0400 Subject: [PATCH 26/75] lint --- frontend/src/pages/ResourcesDashboard/index.js | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 5ae307188d..7c5ca8e0fc 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -198,23 +198,21 @@ export default function ResourcesDashboard() { // Filters passed also contains region. const filterQuery = filtersToQueryString(filtersToApply); try { - -/* const oldData = await fetchResourceData( + /* const oldData = await fetchResourceData( filterQuery, ); */ - //const timeBefore = new Date().getTime(); + // const timeBefore = new Date().getTime(); const data = await fetchFlatResourceData( filterQuery, ); - //const timeAfter = new Date().getTime(); - //const timeTaken = timeAfter - timeBefore; - //alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); - + // const timeAfter = new Date().getTime(); + // const timeTaken = timeAfter - timeBefore; + // alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); // console.log('\n\n\n----- Flat Data: ', data); - //console.log('\n\n\n----- Old Data: ', oldData); + // console.log('\n\n\n----- Old Data: ', oldData); setResourcesData(data); updateError(''); } catch (e) { From affd713bb5314fed8f44bf82b3232fb4476c6295 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 2 Apr 2024 10:08:12 -0400 Subject: [PATCH 27/75] add both routes for now --- .../src/pages/ResourcesDashboard/index.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 7c5ca8e0fc..8c7c99e09b 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -76,6 +76,8 @@ export default function ResourcesDashboard() { (activePage - 1) * REPORTS_PER_PAGE, ); + const [useFlat, setUseFlat] = useState(false); + const getFiltersWithAllRegions = () => { const filtersWithAllRegions = [...allRegionsFilters]; return filtersWithAllRegions; @@ -198,21 +200,16 @@ export default function ResourcesDashboard() { // Filters passed also contains region. const filterQuery = filtersToQueryString(filtersToApply); try { - /* const oldData = await fetchResourceData( + const timeBefore = new Date().getTime(); + const data = useFlat ? await fetchFlatResourceData( filterQuery, - ); - */ - - // const timeBefore = new Date().getTime(); - const data = await fetchFlatResourceData( + ) : await fetchResourceData( filterQuery, ); - // const timeAfter = new Date().getTime(); - // const timeTaken = timeAfter - timeBefore; - // alert(`Time taken to fetch data: ${timeTaken / 1000} seconds`); + const timeAfter = new Date().getTime(); + const timeTaken = timeAfter - timeBefore; + alert(`${useFlat ? 'NEW' : 'OLD'} Fetch: Time taken to fetch data: ${timeTaken / 1000} seconds`); - // console.log('\n\n\n----- Flat Data: ', data); - // console.log('\n\n\n----- Old Data: ', oldData); setResourcesData(data); updateError(''); } catch (e) { From 7f83e797431c5daa63d2ea1ffde6f87c6aa4290b Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Tue, 2 Apr 2024 10:35:13 -0400 Subject: [PATCH 28/75] hook up both paths from UI --- .../src/pages/ResourcesDashboard/index.js | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 8c7c99e09b..0ac4994143 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -11,7 +11,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import PropTypes from 'prop-types'; import { Helmet } from 'react-helmet'; -import { Grid, Alert } from '@trussworks/react-uswds'; +import { Grid, Alert, Radio } from '@trussworks/react-uswds'; import useDeepCompareEffect from 'use-deep-compare-effect'; import FilterPanel from '../../components/filter/FilterPanel'; import { allRegionsUserHasPermissionTo } from '../../permissions'; @@ -22,7 +22,6 @@ import ResourcesDashboardOverview from '../../widgets/ResourcesDashboardOverview import ResourceUse from '../../widgets/ResourceUse'; import { expandFilters, filtersToQueryString, formatDateRange } from '../../utils'; import './index.scss'; -// eslint-disable-next-line no-unused-vars import { fetchResourceData, fetchFlatResourceData } from '../../fetchers/Resources'; import { downloadReports, @@ -222,6 +221,7 @@ export default function ResourcesDashboard() { fetcHResourcesData(); }, [ filtersToApply, + useFlat, ]); const handleDownloadReports = async (setIsDownloading, setDownloadError, url, buttonRef) => { @@ -280,6 +280,12 @@ export default function ResourcesDashboard() { } }; + const fetchChanged = (e) => { + console.log('fetchChanged', e.target, e.target.value === 'on'); + const isFlat = e.target.name === 'fetchResourceMethodFlat' && e.target.value === 'on'; + setUseFlat(isFlat); + }; + return (
@@ -303,7 +309,26 @@ export default function ResourcesDashboard() { )} - +
+ + +
Date: Tue, 2 Apr 2024 11:45:53 -0400 Subject: [PATCH 29/75] set timeout for cache to 10 sec for testing --- src/routes/resources/handlers.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/routes/resources/handlers.js b/src/routes/resources/handlers.js index 2181c2dacf..b23c12905f 100644 --- a/src/routes/resources/handlers.js +++ b/src/routes/resources/handlers.js @@ -25,6 +25,7 @@ export async function getResourcesDashboardData(req, res) { }); }, JSON.parse, + { EX: 10 }, ); res.json(response); @@ -43,6 +44,7 @@ export async function getFlatResourcesDataWithCache(req, res) { return JSON.stringify(data); }, JSON.parse, + { EX: 10 }, ); res.json(response); From 55b3261714baab3fc7644984892382a3aff63892 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 3 Apr 2024 13:35:01 -0400 Subject: [PATCH 30/75] } --- src/services/dashboards/resource.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 97978c51e6..390ab21bbc 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -355,7 +355,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveResources" aror ON aro.id = aror."activityReportObjectiveId" - WHERE aror."sourceFields" = '{resource}' + WHERE aror."sourceFields" && '{resource}' GROUP BY ar.id, aror."resourceId"; -- 3.) Create Resources temp table (only what we need). @@ -421,6 +421,8 @@ async function GenerateFlatTempTables(reportIds, tblNames) { INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; + console.log('\n\n\n---->Flat: ', flatResourceSql); + const transaction = await sequelize.transaction(); // Execute the flat table sql. await sequelize.query( @@ -597,7 +599,7 @@ function getOverview(tblNames, totalReportCount, transaction) { }); // - Reports with Resources Pct - - const pctOfReportsWithResources = sequelize.query(` + const pctOfResourcesSql = ` SELECT count(DISTINCT "activityReportId")::decimal AS "reportsWithResourcesCount", ${totalReportCount}::decimal AS "totalReportsCount", @@ -607,11 +609,14 @@ function getOverview(tblNames, totalReportCount, transaction) { (round(count(DISTINCT "activityReportId")::decimal / ${totalReportCount}::decimal, 4) * 100)::decimal END AS "resourcesPct" FROM ${tblNames.createdAroResourcesTempTableName}; - `, { + `; + const pctOfReportsWithResources = sequelize.query(pctOfResourcesSql, { type: QueryTypes.SELECT, transaction, }); + // console.log('\n\n\n-----> Pct of Reports with Resources: ', pctOfResourcesSql); + // - Number of Reports with ECLKC Resources Pct - const pctOfECKLKCResources = sequelize.query(` WITH eclkc AS ( @@ -686,6 +691,9 @@ export async function resourceFlatData(scopes) { // Get total number of reports. const totalReportCount = reportIds.length; + + // console.log('\n\n\n----> Report Ids: ', reportIds); + // 2.) Create temp table names. const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; From 07d1dda734b87ca4170ee4f96eca75d86b6e3635 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 3 Apr 2024 14:29:38 -0400 Subject: [PATCH 31/75] fix lint --- src/services/dashboards/resource.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 390ab21bbc..557fe2f452 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -421,7 +421,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; - console.log('\n\n\n---->Flat: ', flatResourceSql); + // console.log('\n\n\n---->Flat: ', flatResourceSql); const transaction = await sequelize.transaction(); // Execute the flat table sql. From fd72bd157b6f91673bb8c207786c39eb2d7db358 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Wed, 3 Apr 2024 18:04:10 -0400 Subject: [PATCH 32/75] take of resource limiter --- src/lib/cache.ts | 2 +- src/services/dashboards/resource.js | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 0f27c3695e..25046f9b59 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (!ignoreCache) { + if (false) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 557fe2f452..bb434402ea 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -355,7 +355,6 @@ async function GenerateFlatTempTables(reportIds, tblNames) { ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveResources" aror ON aro.id = aror."activityReportObjectiveId" - WHERE aror."sourceFields" && '{resource}' GROUP BY ar.id, aror."resourceId"; -- 3.) Create Resources temp table (only what we need). @@ -692,8 +691,9 @@ export async function resourceFlatData(scopes) { // Get total number of reports. const totalReportCount = reportIds.length; - // console.log('\n\n\n----> Report Ids: ', reportIds); - + if (reportIds.length === 0) { + reportIds.push({ id: 0 }); + } // 2.) Create temp table names. const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; From e279126b70de1d43ae55c820ecb7baa3c7d54615 Mon Sep 17 00:00:00 2001 From: nvms Date: Thu, 4 Apr 2024 13:15:16 -0400 Subject: [PATCH 33/75] possibly better keyboard interactions --- frontend/src/components/MultiSelect.js | 20 +++++++++++++++-- .../src/components/__tests__/MultiSelect.js | 22 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/MultiSelect.js b/frontend/src/components/MultiSelect.js index 992ad4c975..a64eebb918 100644 --- a/frontend/src/components/MultiSelect.js +++ b/frontend/src/components/MultiSelect.js @@ -20,7 +20,7 @@ through to react-select. If the selected value is not in the options prop the multiselect box will display an empty tag. */ -import React from 'react'; +import React, { useRef } from 'react'; import PropTypes from 'prop-types'; import Select, { components } from 'react-select'; import Creatable from 'react-select/creatable'; @@ -107,6 +107,7 @@ function MultiSelect({ onClick = () => {}, }) { const inputId = `select-${uuidv4()}`; + const selectorRef = useRef(null); /** * unfortunately, given our support for ie11, we can't @@ -159,6 +160,14 @@ function MultiSelect({ } }; + const onKeyDown = (e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + selectorRef.current.focus(); + onClick(); + } + }; + const Selector = canCreate ? Creatable : Select; return ( @@ -167,8 +176,15 @@ function MultiSelect({ const values = value ? getValues(value) : value; return ( // eslint-disable-next-line jsx-a11y/no-static-element-interactions -
{}} data-testid={`${name}-click-container`}> +
{ }, ]); }); + + describe('the div wrapper', () => { + it('forwards space to the Selector, expanding the multiselect', async () => { + render(); + const container = screen.getByTestId('name-click-container'); + container.focus(); + await act(async () => { + userEvent.type(container, '{space}'); + }); + expect(await screen.findByText('one')).toBeVisible(); + }); + it('forwards enter to the Selector, giving it focus', async () => { + render(); + const container = screen.getByTestId('name-click-container'); + container.focus(); + await act(async () => { + userEvent.type(container, '{enter}'); + }); + const selector = container.querySelector('input'); + expect(selector).toHaveFocus(); + }); + }); }); From e937d8e27247acc16a822fcb2f5e2efc059234da Mon Sep 17 00:00:00 2001 From: nvms Date: Thu, 4 Apr 2024 13:16:07 -0400 Subject: [PATCH 34/75] onClick --- frontend/src/components/__tests__/MultiSelect.js | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/components/__tests__/MultiSelect.js b/frontend/src/components/__tests__/MultiSelect.js index 83196e0da0..06845080e8 100644 --- a/frontend/src/components/__tests__/MultiSelect.js +++ b/frontend/src/components/__tests__/MultiSelect.js @@ -41,6 +41,7 @@ describe('MultiSelect', () => { name="name" options={options} required={false} + onClick={() => {}} /> From 11ad181b163c6c84ad1fb8248ae2d8be61531fee Mon Sep 17 00:00:00 2001 From: nvms Date: Thu, 4 Apr 2024 13:36:19 -0400 Subject: [PATCH 35/75] use aria-hidden --- frontend/src/components/MultiSelect.js | 62 ++++++++++--------- .../src/components/__tests__/MultiSelect.js | 9 ++- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/frontend/src/components/MultiSelect.js b/frontend/src/components/MultiSelect.js index a64eebb918..cb89443dfe 100644 --- a/frontend/src/components/MultiSelect.js +++ b/frontend/src/components/MultiSelect.js @@ -183,36 +183,38 @@ function MultiSelect({ // eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex tabIndex={disabled ? 0 : undefined} > - { - if (onItemSelected) { - onItemSelected(event); - } else if (event) { - onChange(event, controllerOnChange); - } else { - controllerOnChange([]); - } - }} - inputId={inputId} - styles={styles(singleRowInput)} - components={{ ...componentReplacements, DropdownIndicator }} - options={options} - isDisabled={disabled} - tabSelectsValue={false} - isClearable={multiSelectOptions.isClearable} - closeMenuOnSelect={multiSelectOptions.closeMenuOnSelect || false} - controlShouldRenderValue={multiSelectOptions.controlShouldRenderValue} - hideSelectedOptions={multiSelectOptions.hideSelectedOptions} - placeholder={placeholderText || ''} - onCreateOption={onCreateOption} - isMulti - required={!!(required)} - /> +
+ { + if (onItemSelected) { + onItemSelected(event); + } else if (event) { + onChange(event, controllerOnChange); + } else { + controllerOnChange([]); + } + }} + inputId={inputId} + styles={styles(singleRowInput)} + components={{ ...componentReplacements, DropdownIndicator }} + options={options} + isDisabled={disabled} + tabSelectsValue={false} + isClearable={multiSelectOptions.isClearable} + closeMenuOnSelect={multiSelectOptions.closeMenuOnSelect || false} + controlShouldRenderValue={multiSelectOptions.controlShouldRenderValue} + hideSelectedOptions={multiSelectOptions.hideSelectedOptions} + placeholder={placeholderText || ''} + onCreateOption={onCreateOption} + isMulti + required={!!(required)} + /> +
); }} diff --git a/frontend/src/components/__tests__/MultiSelect.js b/frontend/src/components/__tests__/MultiSelect.js index 06845080e8..2777b20a5d 100644 --- a/frontend/src/components/__tests__/MultiSelect.js +++ b/frontend/src/components/__tests__/MultiSelect.js @@ -22,7 +22,7 @@ const customOptions = [ describe('MultiSelect', () => { // eslint-disable-next-line react/prop-types - const TestMultiSelect = ({ onSubmit }) => { + const TestMultiSelect = ({ onSubmit, disabled = false }) => { const { control, handleSubmit } = useForm({ defaultValues: { name: [] }, mode: 'all', @@ -42,6 +42,7 @@ describe('MultiSelect', () => { options={options} required={false} onClick={() => {}} + disabled={disabled} /> @@ -181,5 +182,11 @@ describe('MultiSelect', () => { const selector = container.querySelector('input'); expect(selector).toHaveFocus(); }); + it('hides the Selector with aria-hidden when disabled', async () => { + render(); + const container = screen.getByTestId('name-click-container'); + const div = container.querySelector('div'); + expect(div).toHaveAttribute('aria-hidden', 'true'); + }); }); }); From ede534f12af21c69379fea0f12289d4159399070 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 4 Apr 2024 17:25:47 -0400 Subject: [PATCH 36/75] switch to new resource db for now --- .../ResourcesDashboard/__tests__/index.js | 2 +- .../src/pages/ResourcesDashboard/index.js | 42 ++----------------- src/lib/cache.ts | 2 +- src/routes/resources/handlers.js | 2 - src/services/dashboards/resource.js | 19 ++++----- 5 files changed, 13 insertions(+), 54 deletions(-) diff --git a/frontend/src/pages/ResourcesDashboard/__tests__/index.js b/frontend/src/pages/ResourcesDashboard/__tests__/index.js index 7a5cbc39c0..ed6017c918 100644 --- a/frontend/src/pages/ResourcesDashboard/__tests__/index.js +++ b/frontend/src/pages/ResourcesDashboard/__tests__/index.js @@ -29,7 +29,7 @@ const defaultDate = formatDateRange({ }); const defaultDateParam = `startDate.win=${encodeURIComponent(defaultDate)}`; -const resourcesUrl = join('api', 'resources'); +const resourcesUrl = join('api', 'resources/flat'); const resourcesDefault = { resourcesDashboardOverview: { diff --git a/frontend/src/pages/ResourcesDashboard/index.js b/frontend/src/pages/ResourcesDashboard/index.js index 0ac4994143..a2d22f0075 100644 --- a/frontend/src/pages/ResourcesDashboard/index.js +++ b/frontend/src/pages/ResourcesDashboard/index.js @@ -11,7 +11,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import PropTypes from 'prop-types'; import { Helmet } from 'react-helmet'; -import { Grid, Alert, Radio } from '@trussworks/react-uswds'; +import { Grid, Alert } from '@trussworks/react-uswds'; import useDeepCompareEffect from 'use-deep-compare-effect'; import FilterPanel from '../../components/filter/FilterPanel'; import { allRegionsUserHasPermissionTo } from '../../permissions'; @@ -22,7 +22,7 @@ import ResourcesDashboardOverview from '../../widgets/ResourcesDashboardOverview import ResourceUse from '../../widgets/ResourceUse'; import { expandFilters, filtersToQueryString, formatDateRange } from '../../utils'; import './index.scss'; -import { fetchResourceData, fetchFlatResourceData } from '../../fetchers/Resources'; +import { fetchFlatResourceData } from '../../fetchers/Resources'; import { downloadReports, getReportsViaIdPost, @@ -75,8 +75,6 @@ export default function ResourcesDashboard() { (activePage - 1) * REPORTS_PER_PAGE, ); - const [useFlat, setUseFlat] = useState(false); - const getFiltersWithAllRegions = () => { const filtersWithAllRegions = [...allRegionsFilters]; return filtersWithAllRegions; @@ -199,16 +197,9 @@ export default function ResourcesDashboard() { // Filters passed also contains region. const filterQuery = filtersToQueryString(filtersToApply); try { - const timeBefore = new Date().getTime(); - const data = useFlat ? await fetchFlatResourceData( - filterQuery, - ) : await fetchResourceData( + const data = await fetchFlatResourceData( filterQuery, ); - const timeAfter = new Date().getTime(); - const timeTaken = timeAfter - timeBefore; - alert(`${useFlat ? 'NEW' : 'OLD'} Fetch: Time taken to fetch data: ${timeTaken / 1000} seconds`); - setResourcesData(data); updateError(''); } catch (e) { @@ -221,7 +212,6 @@ export default function ResourcesDashboard() { fetcHResourcesData(); }, [ filtersToApply, - useFlat, ]); const handleDownloadReports = async (setIsDownloading, setDownloadError, url, buttonRef) => { @@ -280,12 +270,6 @@ export default function ResourcesDashboard() { } }; - const fetchChanged = (e) => { - console.log('fetchChanged', e.target, e.target.value === 'on'); - const isFlat = e.target.name === 'fetchResourceMethodFlat' && e.target.value === 'on'; - setUseFlat(isFlat); - }; - return (
@@ -309,26 +293,6 @@ export default function ResourcesDashboard() { )} -
- - -
Flat: ', flatResourceSql); - const transaction = await sequelize.transaction(); // Execute the flat table sql. await sequelize.query( @@ -614,8 +613,6 @@ function getOverview(tblNames, totalReportCount, transaction) { transaction, }); - // console.log('\n\n\n-----> Pct of Reports with Resources: ', pctOfResourcesSql); - // - Number of Reports with ECLKC Resources Pct - const pctOfECKLKCResources = sequelize.query(` WITH eclkc AS ( @@ -695,13 +692,13 @@ export async function resourceFlatData(scopes) { reportIds.push({ id: 0 }); } // 2.) Create temp table names. - const createdArTempTableName = `Z_temp_resource_ars__${uuidv4().replaceAll('-', '_')}`; - const createdAroResourcesTempTableName = `Z_temp_resource_aro_resources__${uuidv4().replaceAll('-', '_')}`; - const createdResourcesTempTableName = `Z_temp_resource_resources__${uuidv4().replaceAll('-', '_')}`; - const createdAroTopicsTempTableName = `Z_temp_resource_aro_topics__${uuidv4().replaceAll('-', '_')}`; - const createdTopicsTempTableName = `Z_temp_resource_topics__${uuidv4().replaceAll('-', '_')}`; - const createdFlatResourceHeadersTempTableName = `Z_temp_flat_resources_headers__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. - const createdFlatResourceTempTableName = `Z_temp_flat_resources__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. + const createdArTempTableName = `Z_temp_resdb_ar__${uuidv4().replaceAll('-', '_')}`; + const createdAroResourcesTempTableName = `Z_temp_resdb_aror__${uuidv4().replaceAll('-', '_')}`; + const createdResourcesTempTableName = `Z_temp_resdb_res__${uuidv4().replaceAll('-', '_')}`; + const createdAroTopicsTempTableName = `Z_temp_resdb_arot__${uuidv4().replaceAll('-', '_')}`; + const createdTopicsTempTableName = `Z_temp_resdb_topics__${uuidv4().replaceAll('-', '_')}`; + const createdFlatResourceHeadersTempTableName = `Z_temp_resdb_headers__${uuidv4().replaceAll('-', '_')}`; // Date Headers. + const createdFlatResourceTempTableName = `Z_temp_resdb_flat__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. const tempTableNames = { createdArTempTableName, From 653f5f67d52cef9612767b92b6959dc2c3d8f6c0 Mon Sep 17 00:00:00 2001 From: nvms Date: Fri, 5 Apr 2024 10:18:50 -0400 Subject: [PATCH 37/75] don't prevent default here --- frontend/src/components/MultiSelect.js | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/components/MultiSelect.js b/frontend/src/components/MultiSelect.js index cb89443dfe..868d1541ad 100644 --- a/frontend/src/components/MultiSelect.js +++ b/frontend/src/components/MultiSelect.js @@ -162,7 +162,6 @@ function MultiSelect({ const onKeyDown = (e) => { if (e.key === 'Enter' || e.key === ' ') { - e.preventDefault(); selectorRef.current.focus(); onClick(); } From 112a6dca8f0aba406b4aff0eeba2cb54bfe70bf4 Mon Sep 17 00:00:00 2001 From: nvms Date: Fri, 5 Apr 2024 10:49:56 -0400 Subject: [PATCH 38/75] fix test --- frontend/src/components/__tests__/MultiSelect.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/__tests__/MultiSelect.js b/frontend/src/components/__tests__/MultiSelect.js index 2777b20a5d..6e5da8a87f 100644 --- a/frontend/src/components/__tests__/MultiSelect.js +++ b/frontend/src/components/__tests__/MultiSelect.js @@ -173,7 +173,7 @@ describe('MultiSelect', () => { expect(await screen.findByText('one')).toBeVisible(); }); it('forwards enter to the Selector, giving it focus', async () => { - render(); + render( {}} />); const container = screen.getByTestId('name-click-container'); container.focus(); await act(async () => { From 8965a1b535694b2a4976d24cd1fda560e237ab27 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Fri, 5 Apr 2024 17:18:50 -0400 Subject: [PATCH 39/75] first pass at Garretts suggestions --- src/services/dashboards/resource.js | 80 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index a350501a67..408ae53b77 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -332,8 +332,9 @@ const switchToTopicCentric = (input) => { }; async function GenerateFlatTempTables(reportIds, tblNames) { - const flatResourceSql = ` + const flatResourceSql = /* sql */ ` -- 1.) Create AR temp table. + DROP TABLE IF EXISTS ${tblNames.createdArTempTableName}; SELECT id, "startDate", @@ -346,6 +347,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { WHERE ar."id" IN (${reportIds.map((r) => r.id).join(',')}); -- 2.) Create ARO Resources temp table. + DROP TABLE IF EXISTS ${tblNames.createdAroResourcesTempTableName}; SELECT ar.id AS "activityReportId", aror."resourceId" @@ -359,6 +361,11 @@ async function GenerateFlatTempTables(reportIds, tblNames) { GROUP BY ar.id, aror."resourceId"; -- 3.) Create Resources temp table (only what we need). + DROP TABLE IF EXISTS ${tblNames.createdResourcesTempTableName}; + WITH distinctResources AS ( + SELECT DISTINCT "resourceId" + FROM ${tblNames.createdAroResourcesTempTableName} + ) SELECT id, domain, @@ -366,12 +373,11 @@ async function GenerateFlatTempTables(reportIds, tblNames) { title INTO TEMP ${tblNames.createdResourcesTempTableName} FROM "Resources" - WHERE id IN ( - SELECT DISTINCT "resourceId" - FROM ${tblNames.createdAroResourcesTempTableName} - ); + JOIN distinctResources dr + ON "Resources".id = dr."resourceId"; -- 4.) Create ARO Topics temp table. + DROP TABLE IF EXISTS ${tblNames.createdAroTopicsTempTableName}; SELECT ar.id AS "activityReportId", arot."activityReportObjectiveId", -- We need to group by this incase of multiple aro's. @@ -385,17 +391,21 @@ async function GenerateFlatTempTables(reportIds, tblNames) { GROUP BY ar.id, arot."activityReportObjectiveId", arot."topicId"; -- 5.) Create Topics temp table (only what we need). + DROP TABLE IF EXISTS ${tblNames.createdTopicsTempTableName}; + WITH distinctTopics AS ( + SELECT DISTINCT "topicId" + FROM ${tblNames.createdAroTopicsTempTableName} + ) SELECT id, name INTO TEMP ${tblNames.createdTopicsTempTableName} FROM "Topics" - WHERE id IN ( - SELECT DISTINCT "topicId" - FROM ${tblNames.createdAroTopicsTempTableName} - ); + JOIN distinctTopics dt + ON "Topics".id = dt."topicId"; -- 6.) Create Flat Resource temp table. + DROP TABLE IF EXISTS ${tblNames.createdFlatResourceTempTableName}; SELECT ar.id, ar."startDate", @@ -412,6 +422,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { ON aror."resourceId" = arorr.id; -- 7.) Create date headers. + DROP TABLE IF EXISTS ${tblNames.createdFlatResourceHeadersTempTableName}; SELECT generate_series( date_trunc('month', (SELECT MIN("startDate") FROM ${tblNames.createdFlatResourceTempTableName})), @@ -434,7 +445,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { } function getResourceUseSql(tblNames, transaction) { - const resourceUseSql = ` + const resourceUseSql = /* sql */` WITH urlvals AS ( SELECT url, @@ -472,8 +483,7 @@ function getResourceUseSql(tblNames, transaction) { coalesce(u."resourceCount", 0) AS "resourceCount", t."totalCount" FROM distincturls d - JOIN series s - ON 1=1 + CROSS JOIN series s JOIN totals t ON d.url = t.url LEFT JOIN urlvals u @@ -491,8 +501,8 @@ function getResourceUseSql(tblNames, transaction) { } function getTopicsUseSql(tblNames, transaction) { - const topicUseSql = ` - WITH topics AS ( + const topicUseSql = /* sql */` + WITH topicsuse AS ( SELECT f.id, t.name, @@ -512,7 +522,7 @@ function getTopicsUseSql(tblNames, transaction) { "name", "rollUpDate", SUM("resourceCount") AS "resourceCount" - FROM topics + FROM topicsuse GROUP BY "name", "rollUpDate" ), totals AS @@ -553,7 +563,7 @@ function getTopicsUseSql(tblNames, transaction) { function getOverview(tblNames, totalReportCount, transaction) { // - Number of Participants - - const numberOfParticipants = sequelize.query(` + const numberOfParticipants = sequelize.query(/* sql */` WITH ar_participants AS ( SELECT id, @@ -569,25 +579,14 @@ function getOverview(tblNames, totalReportCount, transaction) { transaction, }); - const numberOfRecipSql = ` - WITH ars AS ( - SELECT - DISTINCT id - FROM ${tblNames.createdFlatResourceTempTableName} f - ), recipients AS ( - SELECT - DISTINCT r.id - FROM ars ar - JOIN "ActivityRecipients" arr - ON ar.id = arr."activityReportId" - JOIN "Grants" g - ON arr."grantId" = g.id AND g."status" = 'Active' - JOIN "Recipients" r - ON g."recipientId" = r.id - ) - SELECT - count(r.id) AS recipients - FROM recipients r; + const numberOfRecipSql = /* sql */` + SELECT + count(DISTINCT g."recipientId") AS recipients + FROM ${tblNames.createdFlatResourceTempTableName} ar + JOIN "ActivityRecipients" arr + ON ar.id = arr."activityReportId" + JOIN "Grants" g + ON arr."grantId" = g.id `; // - Number of Recipients - @@ -597,7 +596,7 @@ function getOverview(tblNames, totalReportCount, transaction) { }); // - Reports with Resources Pct - - const pctOfResourcesSql = ` + const pctOfResourcesSql = /* sql */` SELECT count(DISTINCT "activityReportId")::decimal AS "reportsWithResourcesCount", ${totalReportCount}::decimal AS "totalReportsCount", @@ -614,7 +613,7 @@ function getOverview(tblNames, totalReportCount, transaction) { }); // - Number of Reports with ECLKC Resources Pct - - const pctOfECKLKCResources = sequelize.query(` + const pctOfECKLKCResources = sequelize.query(/* sql */` WITH eclkc AS ( SELECT COUNT(DISTINCT url) AS "eclkcCount" @@ -634,15 +633,14 @@ function getOverview(tblNames, totalReportCount, transaction) { ELSE round(e."eclkcCount" / r."allCount"::decimal * 100,4) END AS "eclkcPct" FROM eclkc e - JOIN allres r - ON 1=1; + CROSS JOIN allres r; `, { type: QueryTypes.SELECT, transaction, }); // 5.) Date Headers table. - const dateHeaders = sequelize.query(` + const dateHeaders = sequelize.query(/* sql */` SELECT to_char("date", 'Mon-YY') AS "rollUpDate" FROM ${tblNames.createdFlatResourceHeadersTempTableName}; @@ -1929,7 +1927,7 @@ Expected JSON: title: 'Jan-22', value: '14', }, - {79 + { title: 'Feb-22', value: '20', }, From 40934a69152e7b0d8485d28c95eec8de955236e3 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Mon, 8 Apr 2024 10:02:44 -0400 Subject: [PATCH 40/75] Create backend for reason list --- src/widgets/helpers.js | 46 +++++- src/widgets/index.js | 2 + src/widgets/reasonList.js | 16 +- src/widgets/trOverview.ts | 34 +--- src/widgets/trReasonList.ts | 25 +++ src/widgets/trReasonlist.test.js | 274 +++++++++++++++++++++++++++++++ src/widgets/types.ts | 6 + 7 files changed, 359 insertions(+), 44 deletions(-) create mode 100644 src/widgets/trReasonList.ts create mode 100644 src/widgets/trReasonlist.test.js create mode 100644 src/widgets/types.ts diff --git a/src/widgets/helpers.js b/src/widgets/helpers.js index 590f95bacb..784dd076c4 100644 --- a/src/widgets/helpers.js +++ b/src/widgets/helpers.js @@ -1,12 +1,56 @@ import { Op } from 'sequelize'; -import { REPORT_STATUSES } from '@ttahub/common'; +import { REPORT_STATUSES, TRAINING_REPORT_STATUSES, REASONS } from '@ttahub/common'; import { ActivityReport, Grant, Recipient, + SessionReportPilot, sequelize, } from '../models'; +export function generateReasonList() { + const reasons = REASONS + .map((reason) => ({ name: reason, count: 0 })) + .sort((a, b) => { + if (a.name < b.name) { + return -1; + } + if (a.name > b.name) { + return 1; + } + return 0; + }); + + return reasons; +} + +export function baseTRScopes(scopes) { + return { + where: { + [Op.and]: [ + { + 'data.status': { + [Op.in]: [ + TRAINING_REPORT_STATUSES.IN_PROGRESS, + TRAINING_REPORT_STATUSES.COMPLETE, + ], + }, + }, + ...scopes.trainingReport, + ], + }, + include: { + model: SessionReportPilot, + as: 'sessionReports', + attributes: ['data', 'eventId'], + where: { + 'data.status': TRAINING_REPORT_STATUSES.COMPLETE, + }, + required: true, + }, + }; +} + export async function getAllRecipientsFiltered(scopes) { return Recipient.findAll({ attributes: [ diff --git a/src/widgets/index.js b/src/widgets/index.js index 015aee1e18..1495cff751 100644 --- a/src/widgets/index.js +++ b/src/widgets/index.js @@ -12,6 +12,7 @@ import goalsByStatus from './regionalGoalDashboard/goalsByStatus'; import goalsPercentage from './regionalGoalDashboard/goalsPercentage'; import topicsByGoalStatus from './regionalGoalDashboard/topicsByGoalStatus'; import trOverview from './trOverview'; +import trReasonList from './trReasonList'; /* All widgets need to be added to this object @@ -22,6 +23,7 @@ export default { dashboardOverview, totalHrsAndRecipientGraph, reasonList, + trReasonList, topicFrequencyGraph, targetPopulationTable, frequencyGraph, diff --git a/src/widgets/reasonList.js b/src/widgets/reasonList.js index 5ee3442d76..589ee40785 100644 --- a/src/widgets/reasonList.js +++ b/src/widgets/reasonList.js @@ -1,7 +1,7 @@ import { Op } from 'sequelize'; -import { REPORT_STATUSES, REASONS } from '@ttahub/common'; +import { REPORT_STATUSES } from '@ttahub/common'; import { ActivityReport } from '../models'; -import { countBySingleKey } from './helpers'; +import { countBySingleKey, generateReasonList } from './helpers'; export default async function reasonList(scopes) { // Query Database for all Reasons within the scope. @@ -18,17 +18,7 @@ export default async function reasonList(scopes) { raw: true, }); - const reasons = REASONS - .map((reason) => ({ name: reason, count: 0 })) - .sort((a, b) => { - if (a.name < b.name) { - return -1; - } - if (a.name > b.name) { - return 1; - } - return 0; - }); + const reasons = generateReasonList(); return countBySingleKey(res, 'reason', reasons); } diff --git a/src/widgets/trOverview.ts b/src/widgets/trOverview.ts index 8b7171237b..896e0b8002 100644 --- a/src/widgets/trOverview.ts +++ b/src/widgets/trOverview.ts @@ -1,14 +1,14 @@ -import { Op, WhereOptions } from 'sequelize'; -import { TRAINING_REPORT_STATUSES } from '@ttahub/common'; +import { Op } from 'sequelize'; import db from '../models'; import { + baseTRScopes, formatNumber, getAllRecipientsFiltered, } from './helpers'; +import { IScopes } from './types'; const { EventReportPilot: TrainingReport, - SessionReportPilot: SessionReport, Recipient, Grant, } = db; @@ -17,11 +17,6 @@ const { * interface for scopes */ -interface IScopes { - grant: WhereOptions[], - trainingReport: WhereOptions[], -} - /** * Interface for the data returned by the Training Report findAll * we use to calculate the data for the TR Overview widget @@ -84,28 +79,7 @@ export default async function trOverview( // Get all completed training reports and their session reports const reports = await TrainingReport.findAll({ attributes: ['data', 'id'], - where: { - [Op.and]: [ - { - 'data.status': { - [Op.in]: [ - TRAINING_REPORT_STATUSES.IN_PROGRESS, - TRAINING_REPORT_STATUSES.COMPLETE, - ], - }, - }, - ...scopes.trainingReport, - ], - }, - include: { - model: SessionReport, - as: 'sessionReports', - attributes: ['data', 'eventId'], - where: { - 'data.status': TRAINING_REPORT_STATUSES.COMPLETE, - }, - required: true, - }, + ...baseTRScopes(scopes), }) as ITrainingReportForOverview[]; const data = reports.reduce((acc: IReportData, report) => { diff --git a/src/widgets/trReasonList.ts b/src/widgets/trReasonList.ts new file mode 100644 index 0000000000..fa96427b5f --- /dev/null +++ b/src/widgets/trReasonList.ts @@ -0,0 +1,25 @@ +import db from '../models'; +import { baseTRScopes, countBySingleKey, generateReasonList } from './helpers'; +import { IScopes } from './types'; + +const { EventReportPilot: TrainingReport } = db; + +export default async function trReasonList(scopes: IScopes = { grant: [], trainingReport: [] }) { + const res = await TrainingReport.findAll({ + attributes: [ + 'data', + 'id', + ], + ...baseTRScopes(scopes), + }) as { + data: { + reasons: string[], + }, + }[]; + + const reasons = generateReasonList(); + + const mapped = res.map((r) => ({ reasons: r.data.reasons })); + + return countBySingleKey(mapped, 'reasons', reasons); +} diff --git a/src/widgets/trReasonlist.test.js b/src/widgets/trReasonlist.test.js new file mode 100644 index 0000000000..976438633a --- /dev/null +++ b/src/widgets/trReasonlist.test.js @@ -0,0 +1,274 @@ +import { TRAINING_REPORT_STATUSES, REASONS } from '@ttahub/common'; +import db, { + EventReportPilot, + SessionReportPilot, + Recipient, + Grant, + User, +} from '../models'; +import { + createUser, + createGrant, + createRecipient, + createSessionReport, + createTrainingReport, +} from '../testUtils'; +import trReasonList from './trReasonList'; + +// We need to mock this so that we don't try to send emails or otherwise engage the queue +jest.mock('bull'); + +describe('TR reason list', () => { + let userCreator; + let userPoc; + let userCollaborator; + + let recipient1; + let recipient2; + let recipient3; + let recipient4; + let recipient5; + + let grant1; + let grant2; + let grant3; + let grant4; + let grant5; + + let trainingReport1; + let trainingReport2; + let trainingReport3; + + beforeAll(async () => { + // user/creator + userCreator = await createUser(); + // user/poc + userPoc = await createUser(); + // user/collaborator ID + userCollaborator = await createUser(); + + // recipient 1 + recipient1 = await createRecipient(); + // recipient 2 + recipient2 = await createRecipient(); + // recipient 3 + recipient3 = await createRecipient(); + // recipient 4 + recipient4 = await createRecipient(); + // recipient 5 (only on uncompleted report) + recipient5 = await createRecipient(); + + // grant 1 + grant1 = await createGrant({ recipientId: recipient1.id, regionId: userCreator.homeRegionId }); + // grant 2 + grant2 = await createGrant({ recipientId: recipient2.id, regionId: userCreator.homeRegionId }); + // grant 3 + grant3 = await createGrant({ recipientId: recipient3.id, regionId: userCreator.homeRegionId }); + // grant 4 + grant4 = await createGrant({ recipientId: recipient4.id, regionId: userCreator.homeRegionId }); + // grant 5 (only on uncompleted report) + grant5 = await createGrant({ recipientId: recipient5.id, regionId: userCreator.homeRegionId }); + + // training report 1 + trainingReport1 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Noncompliance', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 1 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + + // - session report 2 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + + // training report 2 + trainingReport2 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 3 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'hybrid', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 12, + numberOfParticipantsInPerson: 13, + numberOfParticipants: 0, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + + // - session report 4 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant2.id }, { value: grant3.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + + // training report 3 (sessions not completed) + trainingReport3 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + }, { individualHooks: false }); + + // - session report 5 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // - session report 6 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // update TR 1 to complete, the others will be "in progress" as they have sessions + await trainingReport1.update({ + data: { + ...trainingReport1.data, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + }); + + afterAll(async () => { + // delete session reports + await SessionReportPilot.destroy({ + where: { + eventId: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + // delete training reports + await EventReportPilot.destroy({ + where: { + id: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + await db.GrantNumberLink.destroy({ + where: { + grantId: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + force: true, + }); + + // delete grants + await Grant.destroy({ + where: { + id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + }); + + // delete recipients + await Recipient.destroy({ + where: { + id: [recipient1.id, recipient2.id, recipient3.id, recipient4.id, recipient5.id], + }, + }); + + // delete users + await User.destroy({ + where: { + id: [userCreator.id, userPoc.id, userCollaborator.id], + }, + }); + + await db.sequelize.close(); + }); + + it('filters and calculates training report reasons', async () => { + // Confine this to the grants and reports that we created + const scopes = { + grant: [ + { id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id] }, + ], + trainingReport: [ + { id: [trainingReport1.id, trainingReport2.id, trainingReport3.id] }, + ], + }; + + // run our function + const data = await trReasonList(scopes); + + expect(data.length).toBe(REASONS.length); + + const areaOfConcern = data.find((reason) => reason.name === 'Monitoring | Area of Concern'); + expect(areaOfConcern.count).toBe(2); + + const noncompliance = data.find((reason) => reason.name === 'Monitoring | Noncompliance'); + expect(noncompliance.count).toBe(1); + + const deficiency = data.find((reason) => reason.name === 'Monitoring | Deficiency'); + expect(deficiency.count).toBe(2); + + const filteredOut = data.filter((reason) => reason.count === 0); + + expect(filteredOut.length).toBe(REASONS.length - 3); + }); +}); diff --git a/src/widgets/types.ts b/src/widgets/types.ts new file mode 100644 index 0000000000..2a9d70769b --- /dev/null +++ b/src/widgets/types.ts @@ -0,0 +1,6 @@ +import { WhereOptions } from 'sequelize'; + +export interface IScopes { + grant: WhereOptions[], + trainingReport: WhereOptions[], +} From b3493589d16e49af6688d07fe43a1b88873226ac Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 11:16:39 -0400 Subject: [PATCH 41/75] more fixes per Garrett --- src/lib/cache.ts | 2 +- src/services/dashboards/resource.js | 16 ++++++++-------- src/services/dashboards/resourceFlat.test.js | 16 ++++++++++++---- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 0f27c3695e..25046f9b59 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (!ignoreCache) { + if (false) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 408ae53b77..fac76bb38e 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -331,7 +331,7 @@ const switchToTopicCentric = (input) => { }); }; -async function GenerateFlatTempTables(reportIds, tblNames) { +async function GenerateFlatTempTables(reportIds, tblNames, transaction = null) { const flatResourceSql = /* sql */ ` -- 1.) Create AR temp table. DROP TABLE IF EXISTS ${tblNames.createdArTempTableName}; @@ -380,7 +380,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { DROP TABLE IF EXISTS ${tblNames.createdAroTopicsTempTableName}; SELECT ar.id AS "activityReportId", - arot."activityReportObjectiveId", -- We need to group by this incase of multiple aro's. + aro."objectiveId", arot."topicId" INTO TEMP ${tblNames.createdAroTopicsTempTableName} FROM ${tblNames.createdArTempTableName} ar @@ -388,7 +388,7 @@ async function GenerateFlatTempTables(reportIds, tblNames) { ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveTopics" arot ON aro.id = arot."activityReportObjectiveId" - GROUP BY ar.id, arot."activityReportObjectiveId", arot."topicId"; + GROUP BY ar.id, aro."objectiveId", arot."topicId"; -- 5.) Create Topics temp table (only what we need). DROP TABLE IF EXISTS ${tblNames.createdTopicsTempTableName}; @@ -432,7 +432,8 @@ async function GenerateFlatTempTables(reportIds, tblNames) { INTO TEMP ${tblNames.createdFlatResourceHeadersTempTableName}; `; - const transaction = await sequelize.transaction(); + // console.log('\n\n\n---- Flat sql: ', flatResourceSql, '\n\n\n'); + // Execute the flat table sql. await sequelize.query( flatResourceSql, @@ -441,7 +442,6 @@ async function GenerateFlatTempTables(reportIds, tblNames) { transaction, }, ); - return transaction; } function getResourceUseSql(tblNames, transaction) { @@ -661,7 +661,7 @@ function getOverview(tblNames, totalReportCount, transaction) { Create a flat table to calculate the resource data. Use temp tables to ONLY join to the rows we need. If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ -export async function resourceFlatData(scopes) { +export async function resourceFlatData(scopes, transaction = null) { // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -709,7 +709,7 @@ export async function resourceFlatData(scopes) { }; // 3. Generate the base temp tables (used for all calcs). - const transaction = await GenerateFlatTempTables(reportIds, tempTableNames); + await GenerateFlatTempTables(reportIds, tempTableNames, transaction); // 4.) Calculate the resource data. @@ -748,7 +748,7 @@ export async function resourceFlatData(scopes) { dateHeaders, ], ); - transaction.commit(); + // transaction.commit(); // 5.) Restructure Overview. const overView = { diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index f36f6f139a..1f07605286 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -472,7 +472,9 @@ describe('Resources dashboard', () => { it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { resourceUseResult } = await resourceFlatData(scopes); + const transaction = await db.sequelize.transaction(); + const { resourceUseResult } = await resourceFlatData(scopes, transaction); + await transaction.commit(); expect(resourceUseResult).toBeDefined(); expect(resourceUseResult.length).toBe(3); @@ -506,7 +508,9 @@ describe('Resources dashboard', () => { it('resourceTopicUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { topicUseResult } = await resourceFlatData(scopes); + const transaction = await db.sequelize.transaction(); + const { topicUseResult } = await resourceFlatData(scopes, transaction); + await transaction.commit(); expect(topicUseResult).toBeDefined(); expect(topicUseResult).toStrictEqual([ @@ -536,7 +540,9 @@ describe('Resources dashboard', () => { it('overviewFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { overView } = await resourceFlatData(scopes); + const transaction = await db.sequelize.transaction(); + const { overView } = await resourceFlatData(scopes, transaction); + await transaction.commit(); expect(overView).toBeDefined(); const { numberOfParticipants, @@ -576,7 +582,9 @@ describe('Resources dashboard', () => { it('resourceDateHeadersFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const { dateHeaders } = await resourceFlatData(scopes); + const transaction = await db.sequelize.transaction(); + const { dateHeaders } = await resourceFlatData(scopes, transaction); + await transaction.commit(); expect(dateHeaders).toBeDefined(); expect(dateHeaders.length).toBe(1); expect(dateHeaders).toStrictEqual([ From 0323f83ac7910547f878d09051c866fd04317ab4 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Mon, 8 Apr 2024 11:53:07 -0400 Subject: [PATCH 42/75] Work in progress --- src/widgets/helpers.js | 7 + src/widgets/topicFrequencyGraph.js | 7 +- .../trHoursOfTrainingByNationalCenter.test.js | 298 ++++++++++++++++++ .../trHoursOfTrainingByNationalCenter.ts | 65 ++++ src/widgets/trOverview.ts | 2 +- src/widgets/trReasonList.ts | 2 +- src/widgets/trSessionsByTopic.ts | 60 ++++ src/widgets/trSessionsByTopics.test.js | 298 ++++++++++++++++++ 8 files changed, 731 insertions(+), 8 deletions(-) create mode 100644 src/widgets/trHoursOfTrainingByNationalCenter.test.js create mode 100644 src/widgets/trHoursOfTrainingByNationalCenter.ts create mode 100644 src/widgets/trSessionsByTopic.ts create mode 100644 src/widgets/trSessionsByTopics.test.js diff --git a/src/widgets/helpers.js b/src/widgets/helpers.js index 784dd076c4..ffe0b52da8 100644 --- a/src/widgets/helpers.js +++ b/src/widgets/helpers.js @@ -5,9 +5,16 @@ import { Grant, Recipient, SessionReportPilot, + Topic, sequelize, } from '../models'; +export const getAllTopicsForWidget = async () => Topic.findAll({ + attributes: ['id', 'name', 'deletedAt'], + where: { deletedAt: null }, + order: [['name', 'ASC']], +}); + export function generateReasonList() { const reasons = REASONS .map((reason) => ({ name: reason, count: 0 })) diff --git a/src/widgets/topicFrequencyGraph.js b/src/widgets/topicFrequencyGraph.js index 3c0b1642a7..7e7f8b9a2c 100644 --- a/src/widgets/topicFrequencyGraph.js +++ b/src/widgets/topicFrequencyGraph.js @@ -10,6 +10,7 @@ import { sequelize, } from '../models'; import { scopeToWhere } from '../scopes/utils'; +import { getAllTopicsForWidget as getAllTopics } from './helpers'; const getTopicMappings = async () => sequelize.query(` SELECT @@ -22,12 +23,6 @@ WHERE TT."deletedAt" IS NULL OR TT."mapsTo" IS NOT NULL ORDER BY TT."name" `, { type: QueryTypes.SELECT }); -const getAllTopics = async () => Topic.findAll({ - attributes: ['id', 'name', 'deletedAt'], - where: { deletedAt: null }, - order: [['name', 'ASC']], -}); - export async function topicFrequencyGraph(scopes) { const [ topicsAndParticipants, diff --git a/src/widgets/trHoursOfTrainingByNationalCenter.test.js b/src/widgets/trHoursOfTrainingByNationalCenter.test.js new file mode 100644 index 0000000000..f644226b62 --- /dev/null +++ b/src/widgets/trHoursOfTrainingByNationalCenter.test.js @@ -0,0 +1,298 @@ +import faker from '@faker-js/faker'; +import { TRAINING_REPORT_STATUSES } from '@ttahub/common'; +import db, { + EventReportPilot, + SessionReportPilot, + Recipient, + NationalCenter, + Grant, + User, +} from '../models'; +import { + createUser, + createGrant, + createRecipient, + createSessionReport, + createTrainingReport, +} from '../testUtils'; +import trHoursOfTrainingByNationalCenter from './trHoursOfTrainingByNationalCenter'; + +// We need to mock this so that we don't try to send emails or otherwise engage the queue +jest.mock('bull'); + +describe('TR hours of training by national center', () => { + let userCreator; + let userPoc; + let userCollaborator; + + let recipient1; + let recipient2; + let recipient3; + let recipient4; + let recipient5; + + let grant1; + let grant2; + let grant3; + let grant4; + let grant5; + + let trainingReport1; + let trainingReport2; + let trainingReport3; + + let nationalCenter1; + let nationalCenter2; + + beforeAll(async () => { + // user/creator + userCreator = await createUser(); + // user/poc + userPoc = await createUser(); + // user/collaborator ID + userCollaborator = await createUser(); + + // recipient 1 + recipient1 = await createRecipient(); + // recipient 2 + recipient2 = await createRecipient(); + // recipient 3 + recipient3 = await createRecipient(); + // recipient 4 + recipient4 = await createRecipient(); + // recipient 5 (only on uncompleted report) + recipient5 = await createRecipient(); + + // grant 1 + grant1 = await createGrant({ recipientId: recipient1.id, regionId: userCreator.homeRegionId }); + // grant 2 + grant2 = await createGrant({ recipientId: recipient2.id, regionId: userCreator.homeRegionId }); + // grant 3 + grant3 = await createGrant({ recipientId: recipient3.id, regionId: userCreator.homeRegionId }); + // grant 4 + grant4 = await createGrant({ recipientId: recipient4.id, regionId: userCreator.homeRegionId }); + // grant 5 (only on uncompleted report) + grant5 = await createGrant({ recipientId: recipient5.id, regionId: userCreator.homeRegionId }); + + nationalCenter1 = await NationalCenter.create({ + name: faker.word.adjective(3), + }); + + nationalCenter2 = await NationalCenter.create({ + name: faker.word.adjective(4), + }); + + // training report 1 + trainingReport1 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Noncompliance', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 1 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter1.name, + `${nationalCenter2.name} ${userCreator.fullName}`, + ], + }, + }); + + // - session report 2 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter1.name, + nationalCenter2.name, + ], + }, + }); + + // training report 2 + trainingReport2 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 3 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'hybrid', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 12, + numberOfParticipantsInPerson: 13, + numberOfParticipants: 0, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + `${nationalCenter1.name} ${userCreator.fullName}`, + ], + }, + }); + + // - session report 4 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant2.id }, { value: grant3.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter2.name, + ], + }, + }); + + // training report 3 (sessions not completed) + trainingReport3 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + }, { individualHooks: false }); + + // - session report 5 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // - session report 6 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // update TR 1 to complete, the others will be "in progress" as they have sessions + await trainingReport1.update({ + data: { + ...trainingReport1.data, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + }); + + afterAll(async () => { + // delete session reports + await SessionReportPilot.destroy({ + where: { + eventId: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + // delete training reports + await EventReportPilot.destroy({ + where: { + id: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + await db.GrantNumberLink.destroy({ + where: { + grantId: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + force: true, + }); + + // delete grants + await Grant.destroy({ + where: { + id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + }); + + // delete recipients + await Recipient.destroy({ + where: { + id: [recipient1.id, recipient2.id, recipient3.id, recipient4.id, recipient5.id], + }, + }); + + // delete users + await User.destroy({ + where: { + id: [userCreator.id, userPoc.id, userCollaborator.id], + }, + }); + + await NationalCenter.destroy({ + where: { + id: [nationalCenter1.id, nationalCenter2.id], + }, + }); + + await db.sequelize.close(); + }); + + it('filters and calculates hours of training by national center', async () => { + // Confine this to the grants and reports that we created + const scopes = { + grant: [ + { id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id] }, + ], + trainingReport: [ + { id: [trainingReport1.id, trainingReport2.id, trainingReport3.id] }, + ], + }; + + // run our function + const data = await trHoursOfTrainingByNationalCenter(scopes); + + const center1 = data.find((d) => nationalCenter1.name === d.name); + expect(center1.count).toBe(2); + + const center2 = data.find((d) => nationalCenter2.name === d.name); + expect(center2.count).toBe(1); + }); +}); diff --git a/src/widgets/trHoursOfTrainingByNationalCenter.ts b/src/widgets/trHoursOfTrainingByNationalCenter.ts new file mode 100644 index 0000000000..8be8effc8f --- /dev/null +++ b/src/widgets/trHoursOfTrainingByNationalCenter.ts @@ -0,0 +1,65 @@ +import db from '../models'; +import { + baseTRScopes, +} from './helpers'; +import { IScopes } from './types'; + +const { + EventReportPilot: TrainingReport, + NationalCenter, +} = db; + +export default async function trHoursOfTrainingByNationalCenter( + scopes: IScopes, +) { + const [reports, nationalCenters] = await Promise.all([ + TrainingReport.findAll({ + attributes: [ + 'data', + ], + ...baseTRScopes(scopes), + }), + NationalCenter.findAll({ + attributes: [ + 'name', + ], + }), + ]) as [ + { + data: { + eventId: string, + }, + sessionReports: { + data: { + objectiveTrainers: string[], + duration: number, + } + }[] + }[], + { + name: string, + }[], + ]; + + const dataStruct = nationalCenters.map((center: { name: string }) => ({ + name: center.name, + count: 0, + })) as { name: string, count: number }[]; + + const response = reports.reduce((acc, report) => { + const { sessionReports } = report; + sessionReports.forEach((sessionReport) => { + const { objectiveTrainers, duration } = sessionReport.data; + + objectiveTrainers.forEach((trainer) => { + const center = dataStruct.find((c) => trainer.includes(c.name)); + if (center) { + center.count += duration; + } + }); + }); + return acc; + }, dataStruct); + + return response; +} diff --git a/src/widgets/trOverview.ts b/src/widgets/trOverview.ts index 896e0b8002..9009fd7dd7 100644 --- a/src/widgets/trOverview.ts +++ b/src/widgets/trOverview.ts @@ -71,7 +71,7 @@ interface IWidgetData { * @returns IWidgetData */ export default async function trOverview( - scopes: IScopes = { grant: [], trainingReport: [] }, + scopes: IScopes, ): Promise { // get all recipients, matching how they are filtered in the AR overview const allRecipientsFiltered = await getAllRecipientsFiltered(scopes); diff --git a/src/widgets/trReasonList.ts b/src/widgets/trReasonList.ts index fa96427b5f..a75869b931 100644 --- a/src/widgets/trReasonList.ts +++ b/src/widgets/trReasonList.ts @@ -4,7 +4,7 @@ import { IScopes } from './types'; const { EventReportPilot: TrainingReport } = db; -export default async function trReasonList(scopes: IScopes = { grant: [], trainingReport: [] }) { +export default async function trReasonList(scopes: IScopes) { const res = await TrainingReport.findAll({ attributes: [ 'data', diff --git a/src/widgets/trSessionsByTopic.ts b/src/widgets/trSessionsByTopic.ts new file mode 100644 index 0000000000..4c5e815a31 --- /dev/null +++ b/src/widgets/trSessionsByTopic.ts @@ -0,0 +1,60 @@ +import db from '../models'; +import { + baseTRScopes, + getAllTopicsForWidget, +} from './helpers'; +import { IScopes } from './types'; + +const { + EventReportPilot: TrainingReport, +} = db; + +export default async function trSessionByTopic( + scopes: IScopes, +) { + const [reports, topics] = await Promise.all([ + TrainingReport.findAll({ + attributes: [ + 'data', + ], + ...baseTRScopes(scopes), + }), + getAllTopicsForWidget(), + ]) as [ + { + data: { + eventId: string, + }, + sessionReports: { + data: { + objectiveTopics: string[], + } + }[] + }[], + { + name: string, + }[], + ]; + + const dataStruct = topics.map((topic: { name: string }) => ({ + name: topic.name, + count: 0, + })) as { name: string, count: number }[]; + + const response = reports.reduce((acc, report) => { + const { sessionReports } = report; + sessionReports.forEach((sessionReport) => { + const { objectiveTopics } = sessionReport.data; + + objectiveTopics.forEach((topic) => { + const d = dataStruct.find((c) => c.name === topic); + if (d) { + d.count += 1; + } + }); + }); + return acc; + }, dataStruct); + + return response; +} diff --git a/src/widgets/trSessionsByTopics.test.js b/src/widgets/trSessionsByTopics.test.js new file mode 100644 index 0000000000..66f2afb41a --- /dev/null +++ b/src/widgets/trSessionsByTopics.test.js @@ -0,0 +1,298 @@ +import faker from '@faker-js/faker'; +import { TRAINING_REPORT_STATUSES } from '@ttahub/common'; +import db, { + EventReportPilot, + SessionReportPilot, + Recipient, + NationalCenter, + Grant, + User, +} from '../models'; +import { + createUser, + createGrant, + createRecipient, + createSessionReport, + createTrainingReport, +} from '../testUtils'; +import trSessionsByTopic from './trSessionsByTopic'; + +// We need to mock this so that we don't try to send emails or otherwise engage the queue +jest.mock('bull'); + +describe('TR sessions by topic', () => { + let userCreator; + let userPoc; + let userCollaborator; + + let recipient1; + let recipient2; + let recipient3; + let recipient4; + let recipient5; + + let grant1; + let grant2; + let grant3; + let grant4; + let grant5; + + let trainingReport1; + let trainingReport2; + let trainingReport3; + + let nationalCenter1; + let nationalCenter2; + + beforeAll(async () => { + // user/creator + userCreator = await createUser(); + // user/poc + userPoc = await createUser(); + // user/collaborator ID + userCollaborator = await createUser(); + + // recipient 1 + recipient1 = await createRecipient(); + // recipient 2 + recipient2 = await createRecipient(); + // recipient 3 + recipient3 = await createRecipient(); + // recipient 4 + recipient4 = await createRecipient(); + // recipient 5 (only on uncompleted report) + recipient5 = await createRecipient(); + + // grant 1 + grant1 = await createGrant({ recipientId: recipient1.id, regionId: userCreator.homeRegionId }); + // grant 2 + grant2 = await createGrant({ recipientId: recipient2.id, regionId: userCreator.homeRegionId }); + // grant 3 + grant3 = await createGrant({ recipientId: recipient3.id, regionId: userCreator.homeRegionId }); + // grant 4 + grant4 = await createGrant({ recipientId: recipient4.id, regionId: userCreator.homeRegionId }); + // grant 5 (only on uncompleted report) + grant5 = await createGrant({ recipientId: recipient5.id, regionId: userCreator.homeRegionId }); + + nationalCenter1 = await NationalCenter.create({ + name: faker.word.adjective(3), + }); + + nationalCenter2 = await NationalCenter.create({ + name: faker.word.adjective(4), + }); + + // training report 1 + trainingReport1 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Noncompliance', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 1 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter1.name, + `${nationalCenter2.name} ${userCreator.fullName}`, + ], + }, + }); + + // - session report 2 + await createSessionReport({ + eventId: trainingReport1.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter1.name, + nationalCenter2.name, + ], + }, + }); + + // training report 2 + trainingReport2 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + data: { + reasons: [ + 'Monitoring | Area of Concern', + 'Monitoring | Deficiency', + ], + }, + }); + + // - session report 3 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'hybrid', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 12, + numberOfParticipantsInPerson: 13, + numberOfParticipants: 0, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + `${nationalCenter1.name} ${userCreator.fullName}`, + ], + }, + }); + + // - session report 4 + await createSessionReport({ + eventId: trainingReport2.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant2.id }, { value: grant3.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.COMPLETE, + objectiveTrainers: [ + nationalCenter2.name, + ], + }, + }); + + // training report 3 (sessions not completed) + trainingReport3 = await createTrainingReport({ + collaboratorIds: [userCollaborator.id], + pocIds: [userPoc.id], + ownerId: userCreator.id, + }, { individualHooks: false }); + + // - session report 5 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // - session report 6 + await createSessionReport({ + eventId: trainingReport3.id, + data: { + deliveryMethod: 'in-person', + duration: 1, + recipients: [{ value: grant1.id }, { value: grant2.id }], + numberOfParticipantsVirtually: 0, + numberOfParticipantsInPerson: 0, + numberOfParticipants: 25, + status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + }, + }); + + // update TR 1 to complete, the others will be "in progress" as they have sessions + await trainingReport1.update({ + data: { + ...trainingReport1.data, + status: TRAINING_REPORT_STATUSES.COMPLETE, + }, + }); + }); + + afterAll(async () => { + // delete session reports + await SessionReportPilot.destroy({ + where: { + eventId: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + // delete training reports + await EventReportPilot.destroy({ + where: { + id: [trainingReport1.id, trainingReport2.id, trainingReport3.id], + }, + }); + + await db.GrantNumberLink.destroy({ + where: { + grantId: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + force: true, + }); + + // delete grants + await Grant.destroy({ + where: { + id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id], + }, + }); + + // delete recipients + await Recipient.destroy({ + where: { + id: [recipient1.id, recipient2.id, recipient3.id, recipient4.id, recipient5.id], + }, + }); + + // delete users + await User.destroy({ + where: { + id: [userCreator.id, userPoc.id, userCollaborator.id], + }, + }); + + await NationalCenter.destroy({ + where: { + id: [nationalCenter1.id, nationalCenter2.id], + }, + }); + + await db.sequelize.close(); + }); + + it('filters and calculates hours of training by national center', async () => { + // Confine this to the grants and reports that we created + const scopes = { + grant: [ + { id: [grant1.id, grant2.id, grant3.id, grant4.id, grant5.id] }, + ], + trainingReport: [ + { id: [trainingReport1.id, trainingReport2.id, trainingReport3.id] }, + ], + }; + + // run our function + const data = await trSessionsByTopic(scopes); + + const center1 = data.find((d) => nationalCenter1.name === d.name); + expect(center1.count).toBe(2); + + const center2 = data.find((d) => nationalCenter2.name === d.name); + expect(center2.count).toBe(1); + }); +}); From 5db968cd7e59a2f9cb90723de9a42d2dc20e19f3 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Mon, 8 Apr 2024 13:04:57 -0400 Subject: [PATCH 43/75] Finish tests --- .../trHoursOfTrainingByNationalCenter.test.js | 4 +- src/widgets/trSessionsByTopics.test.js | 50 +++++++++---------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/widgets/trHoursOfTrainingByNationalCenter.test.js b/src/widgets/trHoursOfTrainingByNationalCenter.test.js index f644226b62..a3c41e1cac 100644 --- a/src/widgets/trHoursOfTrainingByNationalCenter.test.js +++ b/src/widgets/trHoursOfTrainingByNationalCenter.test.js @@ -290,9 +290,9 @@ describe('TR hours of training by national center', () => { const data = await trHoursOfTrainingByNationalCenter(scopes); const center1 = data.find((d) => nationalCenter1.name === d.name); - expect(center1.count).toBe(2); + expect(center1.count).toBe(3); const center2 = data.find((d) => nationalCenter2.name === d.name); - expect(center2.count).toBe(1); + expect(center2.count).toBe(3); }); }); diff --git a/src/widgets/trSessionsByTopics.test.js b/src/widgets/trSessionsByTopics.test.js index 66f2afb41a..328656a08b 100644 --- a/src/widgets/trSessionsByTopics.test.js +++ b/src/widgets/trSessionsByTopics.test.js @@ -4,7 +4,7 @@ import db, { EventReportPilot, SessionReportPilot, Recipient, - NationalCenter, + Topic, Grant, User, } from '../models'; @@ -41,8 +41,8 @@ describe('TR sessions by topic', () => { let trainingReport2; let trainingReport3; - let nationalCenter1; - let nationalCenter2; + let topic1; + let topic2; beforeAll(async () => { // user/creator @@ -74,12 +74,12 @@ describe('TR sessions by topic', () => { // grant 5 (only on uncompleted report) grant5 = await createGrant({ recipientId: recipient5.id, regionId: userCreator.homeRegionId }); - nationalCenter1 = await NationalCenter.create({ - name: faker.word.adjective(3), + topic1 = await Topic.create({ + name: faker.word.conjunction(5) + faker.word.adjective(3) + faker.word.noun(4), }); - nationalCenter2 = await NationalCenter.create({ - name: faker.word.adjective(4), + topic2 = await Topic.create({ + name: faker.word.conjunction(3) + faker.word.adjective(4) + faker.word.noun(5), }); // training report 1 @@ -107,10 +107,7 @@ describe('TR sessions by topic', () => { numberOfParticipantsInPerson: 0, numberOfParticipants: 25, status: TRAINING_REPORT_STATUSES.COMPLETE, - objectiveTrainers: [ - nationalCenter1.name, - `${nationalCenter2.name} ${userCreator.fullName}`, - ], + objectiveTopics: [], }, }); @@ -125,9 +122,8 @@ describe('TR sessions by topic', () => { numberOfParticipantsInPerson: 0, numberOfParticipants: 25, status: TRAINING_REPORT_STATUSES.COMPLETE, - objectiveTrainers: [ - nationalCenter1.name, - nationalCenter2.name, + objectiveTopics: [ + topic1.name, ], }, }); @@ -156,8 +152,8 @@ describe('TR sessions by topic', () => { numberOfParticipantsInPerson: 13, numberOfParticipants: 0, status: TRAINING_REPORT_STATUSES.COMPLETE, - objectiveTrainers: [ - `${nationalCenter1.name} ${userCreator.fullName}`, + objectiveTopics: [ + topic2.name, ], }, }); @@ -173,9 +169,7 @@ describe('TR sessions by topic', () => { numberOfParticipantsInPerson: 0, numberOfParticipants: 25, status: TRAINING_REPORT_STATUSES.COMPLETE, - objectiveTrainers: [ - nationalCenter2.name, - ], + objectiveTopics: [], }, }); @@ -197,6 +191,10 @@ describe('TR sessions by topic', () => { numberOfParticipantsInPerson: 0, numberOfParticipants: 25, status: TRAINING_REPORT_STATUSES.IN_PROGRESS, + objectiveTopics: [ + topic1.name, + topic2.name, + ], }, }); @@ -266,16 +264,16 @@ describe('TR sessions by topic', () => { }, }); - await NationalCenter.destroy({ + await Topic.destroy({ where: { - id: [nationalCenter1.id, nationalCenter2.id], + id: [topic1.id, topic2.id], }, }); await db.sequelize.close(); }); - it('filters and calculates hours of training by national center', async () => { + it('filters and calculates sessions by topics', async () => { // Confine this to the grants and reports that we created const scopes = { grant: [ @@ -289,10 +287,10 @@ describe('TR sessions by topic', () => { // run our function const data = await trSessionsByTopic(scopes); - const center1 = data.find((d) => nationalCenter1.name === d.name); - expect(center1.count).toBe(2); + const firstTopic = data.find((d) => topic1.name === d.name); + expect(firstTopic.count).toBe(1); - const center2 = data.find((d) => nationalCenter2.name === d.name); - expect(center2.count).toBe(1); + const secondTopic = data.find((d) => topic2.name === d.name); + expect(secondTopic.count).toBe(1); }); }); From 4a6590e61cf640ceadff496b8e75fd58e82a19d2 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 13:46:13 -0400 Subject: [PATCH 44/75] fixes with Garrett --- src/lib/cache.ts | 2 +- src/services/dashboards/resource.js | 104 ++++----- src/services/dashboards/resourceFlat.test.js | 217 ++++++++++--------- 3 files changed, 155 insertions(+), 168 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 25046f9b59..0f27c3695e 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (false) { + if (!ignoreCache) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index fac76bb38e..77261ef996 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -331,7 +331,7 @@ const switchToTopicCentric = (input) => { }); }; -async function GenerateFlatTempTables(reportIds, tblNames, transaction = null) { +async function GenerateFlatTempTables(reportIds, tblNames) { const flatResourceSql = /* sql */ ` -- 1.) Create AR temp table. DROP TABLE IF EXISTS ${tblNames.createdArTempTableName}; @@ -349,6 +349,7 @@ async function GenerateFlatTempTables(reportIds, tblNames, transaction = null) { -- 2.) Create ARO Resources temp table. DROP TABLE IF EXISTS ${tblNames.createdAroResourcesTempTableName}; SELECT + DISTINCT ar.id AS "activityReportId", aror."resourceId" INTO TEMP ${tblNames.createdAroResourcesTempTableName} @@ -357,63 +358,59 @@ async function GenerateFlatTempTables(reportIds, tblNames, transaction = null) { ON ar."id" = aro."activityReportId" JOIN "ActivityReportObjectiveResources" aror ON aro.id = aror."activityReportObjectiveId" - WHERE aror."sourceFields" && '{resource}' - GROUP BY ar.id, aror."resourceId"; + WHERE aror."sourceFields" && '{resource}'; -- 3.) Create Resources temp table (only what we need). DROP TABLE IF EXISTS ${tblNames.createdResourcesTempTableName}; - WITH distinctResources AS ( - SELECT DISTINCT "resourceId" - FROM ${tblNames.createdAroResourcesTempTableName} - ) SELECT + DISTINCT id, domain, url, title INTO TEMP ${tblNames.createdResourcesTempTableName} FROM "Resources" - JOIN distinctResources dr + JOIN ${tblNames.createdAroResourcesTempTableName} dr ON "Resources".id = dr."resourceId"; - -- 4.) Create ARO Topics temp table. + -- 4.) Create ARO Topics temp table. **** Revisit DROP TABLE IF EXISTS ${tblNames.createdAroTopicsTempTableName}; SELECT ar.id AS "activityReportId", - aro."objectiveId", - arot."topicId" + arot."topicId", + count(DISTINCT aro."objectiveId") AS "objectiveCount" INTO TEMP ${tblNames.createdAroTopicsTempTableName} FROM ${tblNames.createdArTempTableName} ar JOIN "ActivityReportObjectives" aro ON ar."id" = aro."activityReportId" + JOIN "ActivityReportObjectiveResources" aror + ON aro.id = aror."activityReportObjectiveId" JOIN "ActivityReportObjectiveTopics" arot ON aro.id = arot."activityReportObjectiveId" - GROUP BY ar.id, aro."objectiveId", arot."topicId"; + GROUP BY ar.id, arot."topicId"; -- 5.) Create Topics temp table (only what we need). DROP TABLE IF EXISTS ${tblNames.createdTopicsTempTableName}; - WITH distinctTopics AS ( - SELECT DISTINCT "topicId" - FROM ${tblNames.createdAroTopicsTempTableName} - ) SELECT + DISTINCT id, name - INTO TEMP ${tblNames.createdTopicsTempTableName} - FROM "Topics" - JOIN distinctTopics dt - ON "Topics".id = dt."topicId"; + INTO TEMP ${tblNames.createdTopicsTempTableName} + FROM "Topics" + JOIN ${tblNames.createdAroTopicsTempTableName} dt + ON "Topics".id = dt."topicId"; -- 6.) Create Flat Resource temp table. DROP TABLE IF EXISTS ${tblNames.createdFlatResourceTempTableName}; SELECT - ar.id, - ar."startDate", - ar."rollUpDate", - arorr.domain, - arorr.title, - arorr.url, - ar."numberOfParticipants" + DISTINCT + ar.id AS "activityReportId", + ar."startDate", + ar."rollUpDate", + arorr.domain, + arorr.title, + arorr.url, + ar."numberOfParticipants" INTO TEMP ${tblNames.createdFlatResourceTempTableName} FROM ${tblNames.createdArTempTableName} ar JOIN ${tblNames.createdAroResourcesTempTableName} aror @@ -439,19 +436,18 @@ async function GenerateFlatTempTables(reportIds, tblNames, transaction = null) { flatResourceSql, { type: QueryTypes.SELECT, - transaction, }, ); } -function getResourceUseSql(tblNames, transaction) { +function getResourceUseSql(tblNames) { const resourceUseSql = /* sql */` WITH urlvals AS ( SELECT url, title, "rollUpDate", - count(id) AS "resourceCount" + count(tf."activityReportId") AS "resourceCount" FROM ${tblNames.createdFlatResourceTempTableName} tf GROUP BY url, title, "rollUpDate" ORDER BY "url", tf."rollUpDate" ASC), @@ -470,10 +466,6 @@ function getResourceUseSql(tblNames, transaction) { ORDER BY SUM("resourceCount") DESC, url ASC LIMIT 10 - ), - series AS - ( - SELECT * FROM ${tblNames.createdFlatResourceHeadersTempTableName} ) SELECT d.url, @@ -483,7 +475,7 @@ function getResourceUseSql(tblNames, transaction) { coalesce(u."resourceCount", 0) AS "resourceCount", t."totalCount" FROM distincturls d - CROSS JOIN series s + CROSS JOIN ${tblNames.createdFlatResourceHeadersTempTableName} s JOIN totals t ON d.url = t.url LEFT JOIN urlvals u @@ -495,16 +487,15 @@ function getResourceUseSql(tblNames, transaction) { resourceUseSql, { type: QueryTypes.SELECT, - transaction, }, ); } -function getTopicsUseSql(tblNames, transaction) { +function getTopicsUseSql(tblNames) { const topicUseSql = /* sql */` WITH topicsuse AS ( SELECT - f.id, + f."activityReportId", t.name, f."rollUpDate", count(DISTINCT f.url) AS "resourceCount" -- Only count each resource once per ar and topic. @@ -512,7 +503,7 @@ function getTopicsUseSql(tblNames, transaction) { JOIN ${tblNames.createdAroTopicsTempTableName} arot ON t.id = arot."topicId" JOIN ${tblNames.createdFlatResourceTempTableName} f - ON arot."activityReportId" = f.id + ON arot."activityReportId" = f."activityReportId" GROUP BY f.id, t.name, f."rollUpDate" ORDER BY t.name, f."rollUpDate" ASC ), @@ -533,10 +524,6 @@ function getTopicsUseSql(tblNames, transaction) { FROM topicsperdate GROUP BY name ORDER BY SUM("resourceCount") DESC - ), - series AS - ( - SELECT * FROM ${tblNames.createdFlatResourceHeadersTempTableName} ) SELECT d.name, @@ -545,7 +532,7 @@ function getTopicsUseSql(tblNames, transaction) { coalesce(t."resourceCount", 0) AS "resourceCount", tt."totalCount" FROM ${tblNames.createdTopicsTempTableName} d - JOIN series s + JOIN ${tblNames.createdFlatResourceHeadersTempTableName} s ON 1=1 JOIN totals tt ON d.name = tt.name @@ -556,27 +543,25 @@ function getTopicsUseSql(tblNames, transaction) { topicUseSql, { type: QueryTypes.SELECT, - transaction, }, ); } -function getOverview(tblNames, totalReportCount, transaction) { +function getOverview(tblNames, totalReportCount) { // - Number of Participants - const numberOfParticipants = sequelize.query(/* sql */` WITH ar_participants AS ( SELECT - id, - "numberOfParticipants" + f."activityReportId", + f."numberOfParticipants" FROM ${tblNames.createdFlatResourceTempTableName} f - GROUP BY id, "numberOfParticipants" + GROUP BY f."activityReportId", f."numberOfParticipants" ) SELECT SUM("numberOfParticipants") AS participants FROM ar_participants; `, { type: QueryTypes.SELECT, - transaction, }); const numberOfRecipSql = /* sql */` @@ -584,7 +569,7 @@ function getOverview(tblNames, totalReportCount, transaction) { count(DISTINCT g."recipientId") AS recipients FROM ${tblNames.createdFlatResourceTempTableName} ar JOIN "ActivityRecipients" arr - ON ar.id = arr."activityReportId" + ON ar."activityReportId" = arr."activityReportId" JOIN "Grants" g ON arr."grantId" = g.id `; @@ -592,7 +577,6 @@ function getOverview(tblNames, totalReportCount, transaction) { // - Number of Recipients - const numberOfRecipients = sequelize.query(numberOfRecipSql, { type: QueryTypes.SELECT, - transaction, }); // - Reports with Resources Pct - @@ -609,7 +593,6 @@ function getOverview(tblNames, totalReportCount, transaction) { `; const pctOfReportsWithResources = sequelize.query(pctOfResourcesSql, { type: QueryTypes.SELECT, - transaction, }); // - Number of Reports with ECLKC Resources Pct - @@ -618,7 +601,7 @@ function getOverview(tblNames, totalReportCount, transaction) { SELECT COUNT(DISTINCT url) AS "eclkcCount" FROM ${tblNames.createdFlatResourceTempTableName} - WHERE url ilike '%eclkc.ohs.acf.hhs.gov%' + WHERE domain = 'eclkc.ohs.acf.hhs.gov' ), allres AS ( SELECT COUNT(DISTINCT url) AS "allCount" @@ -636,7 +619,6 @@ function getOverview(tblNames, totalReportCount, transaction) { CROSS JOIN allres r; `, { type: QueryTypes.SELECT, - transaction, }); // 5.) Date Headers table. @@ -646,7 +628,6 @@ function getOverview(tblNames, totalReportCount, transaction) { FROM ${tblNames.createdFlatResourceHeadersTempTableName}; `, { type: QueryTypes.SELECT, - transaction, }); return { numberOfParticipants, @@ -661,7 +642,7 @@ function getOverview(tblNames, totalReportCount, transaction) { Create a flat table to calculate the resource data. Use temp tables to ONLY join to the rows we need. If over time the amount of data increases and slows again we can cache the flat table a set frequency. */ -export async function resourceFlatData(scopes, transaction = null) { +export async function resourceFlatData(scopes) { // Date to retrieve report data from. const reportCreatedAtDate = '2022-12-01'; @@ -709,15 +690,15 @@ export async function resourceFlatData(scopes, transaction = null) { }; // 3. Generate the base temp tables (used for all calcs). - await GenerateFlatTempTables(reportIds, tempTableNames, transaction); + await GenerateFlatTempTables(reportIds, tempTableNames); // 4.) Calculate the resource data. // -- Resource Use -- - let resourceUseResult = getResourceUseSql(tempTableNames, transaction); + let resourceUseResult = getResourceUseSql(tempTableNames); // -- Topic Use -- - let topicUseResult = getTopicsUseSql(tempTableNames, transaction); + let topicUseResult = getTopicsUseSql(tempTableNames); // -- Overview -- let { @@ -726,7 +707,7 @@ export async function resourceFlatData(scopes, transaction = null) { pctOfReportsWithResources, pctOfECKLKCResources, dateHeaders, - } = getOverview(tempTableNames, totalReportCount, transaction); + } = getOverview(tempTableNames, totalReportCount); // -- Wait for all results -- [ @@ -748,7 +729,6 @@ export async function resourceFlatData(scopes, transaction = null) { dateHeaders, ], ); - // transaction.commit(); // 5.) Restructure Overview. const overView = { diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 1f07605286..656ad79696 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -472,126 +472,133 @@ describe('Resources dashboard', () => { it('resourceUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const transaction = await db.sequelize.transaction(); - const { resourceUseResult } = await resourceFlatData(scopes, transaction); - await transaction.commit(); - expect(resourceUseResult).toBeDefined(); - expect(resourceUseResult.length).toBe(3); + let resourceUseResult; + db.sequelize.transaction(async () => { + ({ resourceUseResult } = await resourceFlatData(scopes)); - expect(resourceUseResult).toStrictEqual([ - { - date: '2021-01-01', - url: 'https://eclkc.ohs.acf.hhs.gov/test', - rollUpDate: 'Jan-21', - title: null, - resourceCount: '2', - totalCount: '2', - }, - { - date: '2021-01-01', - url: 'https://eclkc.ohs.acf.hhs.gov/test2', - rollUpDate: 'Jan-21', - title: null, - resourceCount: '3', - totalCount: '3', - }, - { - date: '2021-01-01', - url: 'https://non.test1.gov/a/b/c', - rollUpDate: 'Jan-21', - title: null, - resourceCount: '2', - totalCount: '2', - }, - ]); + expect(resourceUseResult).toBeDefined(); + expect(resourceUseResult.length).toBe(3); + + expect(resourceUseResult).toStrictEqual([ + { + date: '2021-01-01', + url: 'https://eclkc.ohs.acf.hhs.gov/test', + rollUpDate: 'Jan-21', + title: null, + resourceCount: '2', + totalCount: '2', + }, + { + date: '2021-01-01', + url: 'https://eclkc.ohs.acf.hhs.gov/test2', + rollUpDate: 'Jan-21', + title: null, + resourceCount: '3', + totalCount: '3', + }, + { + date: '2021-01-01', + url: 'https://non.test1.gov/a/b/c', + rollUpDate: 'Jan-21', + title: null, + resourceCount: '2', + totalCount: '2', + }, + ]); + }); }); it('resourceTopicUseFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const transaction = await db.sequelize.transaction(); - const { topicUseResult } = await resourceFlatData(scopes, transaction); - await transaction.commit(); - expect(topicUseResult).toBeDefined(); + let topicUseResult; + db.sequelize.transaction(async () => { + ({ topicUseResult } = await resourceFlatData(scopes)); - expect(topicUseResult).toStrictEqual([ - { - name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', - }, - { - name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', - }, - { - name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', date: '2021-01-01', - }, - { - name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', - }, - { - name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', - }, - { - name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', - }, - { - name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', - }, - ]); + expect(topicUseResult).toBeDefined(); + + expect(topicUseResult).toStrictEqual([ + { + name: 'CLASS: Classroom Organization', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', + }, + { + name: 'Coaching', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', + }, + { + name: 'ERSEA', rollUpDate: 'Jan-21', resourceCount: '3', totalCount: '3', date: '2021-01-01', + }, + { + name: 'Facilities', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', + }, + { + name: 'Fiscal / Budget', rollUpDate: 'Jan-21', resourceCount: '1', totalCount: '1', date: '2021-01-01', + }, + { + name: 'Nutrition', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', + }, + { + name: 'Oral Health', rollUpDate: 'Jan-21', resourceCount: '2', totalCount: '2', date: '2021-01-01', + }, + ]); + }); }); it('overviewFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const transaction = await db.sequelize.transaction(); - const { overView } = await resourceFlatData(scopes, transaction); - await transaction.commit(); - expect(overView).toBeDefined(); - const { - numberOfParticipants, - numberOfRecipients, - pctOfReportsWithResources, - pctOfECKLKCResources, - } = overView; - - // Number of Participants. - expect(numberOfParticipants).toStrictEqual([{ - participants: '44', - }]); - - // Number of Recipients. - expect(numberOfRecipients).toStrictEqual([{ - recipients: '1', - }]); - - // Percent of Reports with Resources. - expect(pctOfReportsWithResources).toStrictEqual([ - { - reportsWithResourcesCount: '4', - totalReportsCount: '5', - resourcesPct: '80.0000', - }, - ]); + let overView; + db.sequelize.transaction(async () => { + ({ overView } = await resourceFlatData(scopes)); + + expect(overView).toBeDefined(); + const { + numberOfParticipants, + numberOfRecipients, + pctOfReportsWithResources, + pctOfECKLKCResources, + } = overView; + + // Number of Participants. + expect(numberOfParticipants).toStrictEqual([{ + participants: '44', + }]); + + // Number of Recipients. + expect(numberOfRecipients).toStrictEqual([{ + recipients: '1', + }]); + + // Percent of Reports with Resources. + expect(pctOfReportsWithResources).toStrictEqual([ + { + reportsWithResourcesCount: '4', + totalReportsCount: '5', + resourcesPct: '80.0000', + }, + ]); - // Percent of ECLKC reports. - expect(pctOfECKLKCResources).toStrictEqual([ - { - eclkcCount: '2', - allCount: '3', - eclkcPct: '66.6667', - }, - ]); + // Percent of ECLKC reports. + expect(pctOfECKLKCResources).toStrictEqual([ + { + eclkcCount: '2', + allCount: '3', + eclkcPct: '66.6667', + }, + ]); + }); }); it('resourceDateHeadersFlat', async () => { const scopes = await filtersToScopes({ 'region.in': [REGION_ID], 'startDate.win': '2021/01/01-2021/01/31' }); - const transaction = await db.sequelize.transaction(); - const { dateHeaders } = await resourceFlatData(scopes, transaction); - await transaction.commit(); - expect(dateHeaders).toBeDefined(); - expect(dateHeaders.length).toBe(1); - expect(dateHeaders).toStrictEqual([ - { - rollUpDate: 'Jan-21', - }, - ]); + let dateHeaders; + db.sequelize.transaction(async () => { + ({ dateHeaders } = await resourceFlatData(scopes)); + expect(dateHeaders).toBeDefined(); + expect(dateHeaders.length).toBe(1); + expect(dateHeaders).toStrictEqual([ + { + rollUpDate: 'Jan-21', + }, + ]); + }); }); it('should roll up resource use results correctly', async () => { From 6c2cdb3f5bf9a0c88dc0c40529446d70ec6bc17a Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 16:34:16 -0400 Subject: [PATCH 45/75] more updates per Garretts comments --- src/services/dashboards/resource.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/services/dashboards/resource.js b/src/services/dashboards/resource.js index 77261ef996..cb7ec44235 100644 --- a/src/services/dashboards/resource.js +++ b/src/services/dashboards/resource.js @@ -504,7 +504,7 @@ function getTopicsUseSql(tblNames) { ON t.id = arot."topicId" JOIN ${tblNames.createdFlatResourceTempTableName} f ON arot."activityReportId" = f."activityReportId" - GROUP BY f.id, t.name, f."rollUpDate" + GROUP BY f."activityReportId", t.name, f."rollUpDate" ORDER BY t.name, f."rollUpDate" ASC ), topicsperdate AS @@ -587,7 +587,7 @@ function getOverview(tblNames, totalReportCount) { CASE WHEN ${totalReportCount} = 0 THEN 0 ELSE - (round(count(DISTINCT "activityReportId")::decimal / ${totalReportCount}::decimal, 4) * 100)::decimal + (count(DISTINCT "activityReportId") / ${totalReportCount}::decimal * 100)::decimal(5,2) END AS "resourcesPct" FROM ${tblNames.createdAroResourcesTempTableName}; `; @@ -613,7 +613,7 @@ function getOverview(tblNames, totalReportCount) { CASE WHEN r."allCount" = 0 THEN 0 - ELSE round(e."eclkcCount" / r."allCount"::decimal * 100,4) + ELSE (e."eclkcCount" / r."allCount"::decimal * 100)::decimal(5,2) END AS "eclkcPct" FROM eclkc e CROSS JOIN allres r; @@ -671,13 +671,14 @@ export async function resourceFlatData(scopes) { reportIds.push({ id: 0 }); } // 2.) Create temp table names. - const createdArTempTableName = `Z_temp_resdb_ar__${uuidv4().replaceAll('-', '_')}`; - const createdAroResourcesTempTableName = `Z_temp_resdb_aror__${uuidv4().replaceAll('-', '_')}`; - const createdResourcesTempTableName = `Z_temp_resdb_res__${uuidv4().replaceAll('-', '_')}`; - const createdAroTopicsTempTableName = `Z_temp_resdb_arot__${uuidv4().replaceAll('-', '_')}`; - const createdTopicsTempTableName = `Z_temp_resdb_topics__${uuidv4().replaceAll('-', '_')}`; - const createdFlatResourceHeadersTempTableName = `Z_temp_resdb_headers__${uuidv4().replaceAll('-', '_')}`; // Date Headers. - const createdFlatResourceTempTableName = `Z_temp_resdb_flat__${uuidv4().replaceAll('-', '_')}`; // Main Flat Table. + const uuid = uuidv4().replaceAll('-', '_'); + const createdArTempTableName = `Z_temp_resdb_ar__${uuid}`; + const createdAroResourcesTempTableName = `Z_temp_resdb_aror__${uuid}`; + const createdResourcesTempTableName = `Z_temp_resdb_res__${uuid}`; + const createdAroTopicsTempTableName = `Z_temp_resdb_arot__${uuid}`; + const createdTopicsTempTableName = `Z_temp_resdb_topics__${uuid}`; + const createdFlatResourceHeadersTempTableName = `Z_temp_resdb_headers__${uuid}`; // Date Headers. + const createdFlatResourceTempTableName = `Z_temp_resdb_flat__${uuid}`; // Main Flat Table. const tempTableNames = { createdArTempTableName, From 06dd9447f15bfd20fa13776c0c757f61e16c3881 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 17:19:25 -0400 Subject: [PATCH 46/75] add a check to make sure we dont count topics without resources --- src/lib/cache.ts | 2 +- src/services/dashboards/resourceFlat.test.js | 37 +++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 0f27c3695e..25046f9b59 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (!ignoreCache) { + if (false) { redisClient = createClient({ url: redisUrl, socket: { diff --git a/src/services/dashboards/resourceFlat.test.js b/src/services/dashboards/resourceFlat.test.js index 656ad79696..89d1ed844c 100644 --- a/src/services/dashboards/resourceFlat.test.js +++ b/src/services/dashboards/resourceFlat.test.js @@ -137,9 +137,12 @@ let grant; let goal; let objective; let goalTwo; +let goalThree; let objectiveTwo; +let objectiveThree; let activityReportOneObjectiveOne; let activityReportOneObjectiveTwo; +let activityReportOneObjectiveThree; // Topic but no resources. let activityReportObjectiveTwo; let activityReportObjectiveThree; let arIds; @@ -155,6 +158,7 @@ describe('Resources dashboard', () => { }); [goal] = await Goal.findOrCreate({ where: mockGoal, validate: true, individualHooks: true }); [goalTwo] = await Goal.findOrCreate({ where: { ...mockGoal, name: 'Goal 2' }, validate: true, individualHooks: true }); + [goalThree] = await Goal.findOrCreate({ where: { ...mockGoal, name: 'Goal 3' }, validate: true, individualHooks: true }); [objective] = await Objective.findOrCreate({ where: { title: 'Objective 1', @@ -171,6 +175,14 @@ describe('Resources dashboard', () => { }, }); + [objectiveThree] = await Objective.findOrCreate({ + where: { + title: 'Objective 3', + goalId: goalThree.dataValues.id, + status: 'In Progress', + }, + }); + // Get topic ID's. const { topicId: classOrgTopicId } = await Topic.findOne({ attributes: [['id', 'topicId']], @@ -288,6 +300,23 @@ describe('Resources dashboard', () => { [ECLKC_RESOURCE_URL, NONECLKC_RESOURCE_URL], ); + // Report 1 - Activity Report Objective 3 (No resources) + // This topic should NOT count as there are no resources. + [activityReportOneObjectiveThree] = await ActivityReportObjective.findOrCreate({ + where: { + activityReportId: reportOne.id, + status: 'Complete', + objectiveId: objectiveThree.id, + }, + }); + + await ActivityReportObjectiveTopic.findOrCreate({ + where: { + activityReportObjectiveId: activityReportOneObjectiveThree.id, + topicId: nutritionTopicId, + }, + }); + // Report 2 (Only ECLKC). const reportTwo = await ActivityReport.create({ ...regionOneReportB }); await ActivityRecipient.create({ activityReportId: reportTwo.id, grantId: mockGrant.id }); @@ -370,7 +399,7 @@ describe('Resources dashboard', () => { }, }); - // Report 3 Non-ECLKC Resource 1. + // Report 4 Non-ECLKC Resource 1. await processActivityReportObjectiveForResourcesById( activityReportObjectiveForReport4.id, [ECLKC_RESOURCE_URL2], @@ -456,10 +485,10 @@ describe('Resources dashboard', () => { }); // eslint-disable-next-line max-len - await ActivityReportObjective.destroy({ where: { objectiveId: [objective.id, objectiveTwo.id] } }); + await ActivityReportObjective.destroy({ where: { objectiveId: [objective.id, objectiveTwo.id, objectiveThree.id] } }); await ActivityReport.destroy({ where: { id: ids } }); - await Objective.destroy({ where: { id: [objective.id, objectiveTwo.id] }, force: true }); - await Goal.destroy({ where: { id: [goal.id, goalTwo.id] }, force: true }); + await Objective.destroy({ where: { id: [objective.id, objectiveTwo.id, objectiveThree.id] }, force: true }); + await Goal.destroy({ where: { id: [goal.id, goalTwo.id, goalThree.id] }, force: true }); await Grant.destroy({ where: { id: GRANT_ID_ONE }, individualHooks: true }); await User.destroy({ where: { id: [mockUser.id] } }); await Recipient.destroy({ where: { id: RECIPIENT_ID } }); From de6e962f195a6175207cc5ee8e79884d7f8bbaf4 Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 17:36:26 -0400 Subject: [PATCH 47/75] put back cache --- src/lib/cache.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/cache.ts b/src/lib/cache.ts index 25046f9b59..0f27c3695e 100644 --- a/src/lib/cache.ts +++ b/src/lib/cache.ts @@ -48,7 +48,7 @@ export default async function getCachedResponse( let response: string | null = null; try { - if (false) { + if (!ignoreCache) { redisClient = createClient({ url: redisUrl, socket: { From 464386d4c2179353108b7785b916d906fe7857fb Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Mon, 8 Apr 2024 17:47:03 -0400 Subject: [PATCH 48/75] add Matts fix for ar on resource db --- src/routes/activityReports/handlers.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/routes/activityReports/handlers.js b/src/routes/activityReports/handlers.js index b57e099e06..35f66e021e 100644 --- a/src/routes/activityReports/handlers.js +++ b/src/routes/activityReports/handlers.js @@ -769,11 +769,18 @@ export async function getReportsByManyIds(req, res) { try { const userId = await currentUserId(req, res); - const { reportIds } = req.body; + const { + reportIds, offset, sortBy, sortDir, limit, + } = req.body; // this will return a query with region parameters based // on the req user's permissions - const query = await setReadRegions({}, userId); + const query = await setReadRegions({ + offset, + sortBy, + sortDir, + limit, + }, userId); const reportsWithCount = await activityReports(query, false, userId, reportIds); if (!reportsWithCount) { From 4820d6f77837d96e93beae2dcdeb0f2852114c8d Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 08:15:47 -0400 Subject: [PATCH 49/75] add frontend to regional dashboard --- frontend/src/App.scss | 7 +- .../components/TrainingReportDashboard.js | 21 ++- frontend/src/widgets/ReasonList.js | 14 +- .../TRHoursOfTrainingByNationalCenter.js | 4 + frontend/src/widgets/TRReasonList.js | 4 + frontend/src/widgets/VBarGraph.js | 175 ++++++++++++++++++ src/widgets/index.js | 10 +- 7 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 frontend/src/widgets/TRHoursOfTrainingByNationalCenter.js create mode 100644 frontend/src/widgets/TRReasonList.js create mode 100644 frontend/src/widgets/VBarGraph.js diff --git a/frontend/src/App.scss b/frontend/src/App.scss index 61621e0e37..6e86695abb 100644 --- a/frontend/src/App.scss +++ b/frontend/src/App.scss @@ -403,4 +403,9 @@ fill: #1B1B1B; .desktop\:maxw-6 { max-width: 3rem; } -} \ No newline at end of file +} + +.smart-hub--vertical-text { + writing-mode: vertical-lr; + transform: rotate(180deg); +} diff --git a/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js b/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js index e6cc485d7a..1297086797 100644 --- a/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js +++ b/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js @@ -2,6 +2,8 @@ import React from 'react'; import { Helmet } from 'react-helmet'; import { Grid, GridContainer } from '@trussworks/react-uswds'; import Overview from '../../../widgets/TrainingReportDashboardOverview'; +import TRReasonList from '../../../widgets/TRReasonList'; +import TRHoursOfTrainingByNationalCenter from '../../../widgets/TRHoursOfTrainingByNationalCenter'; export default function TrainingReportDashboard() { return ( @@ -16,8 +18,23 @@ export default function TrainingReportDashboard() { loading={false} /> - - + + + + + + diff --git a/frontend/src/widgets/ReasonList.js b/frontend/src/widgets/ReasonList.js index 1366056daf..806609343b 100644 --- a/frontend/src/widgets/ReasonList.js +++ b/frontend/src/widgets/ReasonList.js @@ -19,20 +19,22 @@ const renderReasonList = (data) => { return null; }; -function ReasonList({ data, loading }) { +export function ReasonListTable({ + data, loading, title, +}) { return ( ); } -ReasonList.propTypes = { +ReasonListTable.propTypes = { data: PropTypes.oneOfType([ PropTypes.arrayOf( PropTypes.shape({ @@ -42,10 +44,12 @@ ReasonList.propTypes = { ), PropTypes.shape({}), ]), loading: PropTypes.bool.isRequired, + title: PropTypes.string, }; -ReasonList.defaultProps = { +ReasonListTable.defaultProps = { data: [], + title: 'Reasons in Activity Reports', }; -export default withWidgetData(ReasonList, 'reasonList'); +export default withWidgetData(ReasonListTable, 'reasonList'); diff --git a/frontend/src/widgets/TRHoursOfTrainingByNationalCenter.js b/frontend/src/widgets/TRHoursOfTrainingByNationalCenter.js new file mode 100644 index 0000000000..250bd127e6 --- /dev/null +++ b/frontend/src/widgets/TRHoursOfTrainingByNationalCenter.js @@ -0,0 +1,4 @@ +import VBarGraph from './VBarGraph'; +import withWidgetData from './withWidgetData'; + +export default withWidgetData(VBarGraph, 'trHoursOfTrainingByNationalCenter'); diff --git a/frontend/src/widgets/TRReasonList.js b/frontend/src/widgets/TRReasonList.js new file mode 100644 index 0000000000..9d1ac1a25c --- /dev/null +++ b/frontend/src/widgets/TRReasonList.js @@ -0,0 +1,4 @@ +import { ReasonListTable } from './ReasonList'; +import withWidgetData from './withWidgetData'; + +export default withWidgetData(ReasonListTable, 'trReasonList'); diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js new file mode 100644 index 0000000000..6c0ff00998 --- /dev/null +++ b/frontend/src/widgets/VBarGraph.js @@ -0,0 +1,175 @@ +import React, { useState, useEffect, useRef } from 'react'; +import PropTypes from 'prop-types'; +import { Grid } from '@trussworks/react-uswds'; +// https://github.com/plotly/react-plotly.js/issues/135#issuecomment-501398125 +import Plotly from 'plotly.js-basic-dist'; +import createPlotlyComponent from 'react-plotly.js/factory'; +import colors from '../colors'; +import Container from '../components/Container'; +import AccessibleWidgetData from './AccessibleWidgetData'; +import MediaCaptureButton from '../components/MediaCaptureButton'; + +const Plot = createPlotlyComponent(Plotly); + +function VBarGraph({ + data, + yAxisLabel, + xAxisLabel, + title, + subtitle, + loading, + loadingLabel, +}) { + const [plot, updatePlot] = useState({}); + const bars = useRef(null); + const [showAccessibleData, updateShowAccessibleData] = useState(false); + // toggle the data table + function toggleAccessibleData() { + updateShowAccessibleData((current) => !current); + } + + useEffect(() => { + if (!data || !Array.isArray(data)) { + return; + } + + const names = []; + const counts = []; + + data.forEach((dataPoint) => { + names.push(dataPoint.name); + counts.push(dataPoint.count); + }); + + const trace = { + type: 'bar', + x: names, + y: counts, + hoverinfo: 'y', + marker: { + color: colors.ttahubMediumBlue, + }, + }; + + const layout = { + bargap: 0.5, + height: 300, + hoverlabel: { + bgcolor: '#000', + bordercolor: '#000', + font: { + color: '#fff', + size: 16, + }, + }, + font: { + color: '#1b1b1b', + }, + margin: { + l: 80, + pad: 20, + t: 24, + }, + xaxis: { + automargin: true, + fixedrange: true, + tickangle: 0, + }, + yaxis: { + tickformat: ',.0d', + fixedrange: true, + }, + hovermode: 'none', + }; + + updatePlot({ + data: [trace], + layout, + config: { + responsive: true, displayModeBar: false, hovermode: 'none', + }, + }); + }, [data]); + + return ( + + + +

+ {title} +

+

{subtitle}

+ + + + + + + { showAccessibleData + ? ( + + ) + : ( + <> +
+
+ { yAxisLabel } +
+ + + +
+
+ { xAxisLabel } +
+ + )} + + + ); +} + +VBarGraph.propTypes = { + data: PropTypes.arrayOf( + PropTypes.shape({ + category: PropTypes.string, + count: PropTypes.number, + }), + ), + yAxisLabel: PropTypes.string.isRequired, + xAxisLabel: PropTypes.string.isRequired, + title: PropTypes.string, + subtitle: PropTypes.string, + loading: PropTypes.bool, + loadingLabel: PropTypes.string, +}; + +VBarGraph.defaultProps = { + data: [], + title: 'Vertical Bar Graph', + subtitle: '', + loading: false, + loadingLabel: 'Vertical Bar Graph Loading', +}; + +export default VBarGraph; diff --git a/src/widgets/index.js b/src/widgets/index.js index 1495cff751..5459f0b09b 100644 --- a/src/widgets/index.js +++ b/src/widgets/index.js @@ -13,17 +13,18 @@ import goalsPercentage from './regionalGoalDashboard/goalsPercentage'; import topicsByGoalStatus from './regionalGoalDashboard/topicsByGoalStatus'; import trOverview from './trOverview'; import trReasonList from './trReasonList'; +import trSessionsByTopic from './trSessionsByTopic'; +import trHoursOfTrainingByNationalCenter from './trHoursOfTrainingByNationalCenter'; /* All widgets need to be added to this object */ export default { overview, - trOverview, dashboardOverview, totalHrsAndRecipientGraph, reasonList, - trReasonList, + topicFrequencyGraph, targetPopulationTable, frequencyGraph, @@ -32,4 +33,9 @@ export default { goalsByStatus, goalsPercentage, topicsByGoalStatus, + + trOverview, + trReasonList, + trSessionsByTopic, + trHoursOfTrainingByNationalCenter, }; From 7897e6c39acd6ad7afe3f0147fafde37a1625d73 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 09:32:52 -0400 Subject: [PATCH 50/75] Add widgets to export rollup and comment --- src/widgets/index.js | 10 ++++++++-- src/widgets/trHoursOfTrainingByNationalCenter.ts | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/widgets/index.js b/src/widgets/index.js index 1495cff751..5459f0b09b 100644 --- a/src/widgets/index.js +++ b/src/widgets/index.js @@ -13,17 +13,18 @@ import goalsPercentage from './regionalGoalDashboard/goalsPercentage'; import topicsByGoalStatus from './regionalGoalDashboard/topicsByGoalStatus'; import trOverview from './trOverview'; import trReasonList from './trReasonList'; +import trSessionsByTopic from './trSessionsByTopic'; +import trHoursOfTrainingByNationalCenter from './trHoursOfTrainingByNationalCenter'; /* All widgets need to be added to this object */ export default { overview, - trOverview, dashboardOverview, totalHrsAndRecipientGraph, reasonList, - trReasonList, + topicFrequencyGraph, targetPopulationTable, frequencyGraph, @@ -32,4 +33,9 @@ export default { goalsByStatus, goalsPercentage, topicsByGoalStatus, + + trOverview, + trReasonList, + trSessionsByTopic, + trHoursOfTrainingByNationalCenter, }; diff --git a/src/widgets/trHoursOfTrainingByNationalCenter.ts b/src/widgets/trHoursOfTrainingByNationalCenter.ts index 8be8effc8f..7ba8b26bef 100644 --- a/src/widgets/trHoursOfTrainingByNationalCenter.ts +++ b/src/widgets/trHoursOfTrainingByNationalCenter.ts @@ -52,6 +52,9 @@ export default async function trHoursOfTrainingByNationalCenter( const { objectiveTrainers, duration } = sessionReport.data; objectiveTrainers.forEach((trainer) => { + // trainers were originally and are now stored by the national center abbrev. + // but looking at the data, there was a period where they were stored as + // abbrev - user name, so we need to check for that const center = dataStruct.find((c) => trainer.includes(c.name)); if (center) { center.count += duration; From c0c67eb4657f4f79ad807bf2d6fb7d4cf705e12b Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 13:11:39 -0400 Subject: [PATCH 51/75] Implement frontend --- frontend/src/App.scss | 2 + frontend/src/components/WidgetH2.js | 19 +++++ .../components/TrainingReportDashboard.js | 10 ++- .../src/pages/RegionalDashboard/index.css | 2 +- frontend/src/widgets/BarGraph.js | 4 +- frontend/src/widgets/TableWidget.js | 5 +- frontend/src/widgets/TopicFrequencyGraph.js | 42 +++-------- frontend/src/widgets/VBarGraph.js | 70 +++++++++++-------- frontend/src/widgets/VTopicFrequency.js | 4 ++ .../widgets/__tests__/TopicFrequencyGraph.js | 10 +-- frontend/src/widgets/widgets.scss | 17 +++++ 11 files changed, 107 insertions(+), 78 deletions(-) create mode 100644 frontend/src/components/WidgetH2.js create mode 100644 frontend/src/widgets/VTopicFrequency.js create mode 100644 frontend/src/widgets/widgets.scss diff --git a/frontend/src/App.scss b/frontend/src/App.scss index 6e86695abb..5bf1d6443a 100644 --- a/frontend/src/App.scss +++ b/frontend/src/App.scss @@ -1,6 +1,8 @@ @use 'colors.scss' as *; @use './Grid.scss'; +@use './widgets/widgets.scss'; + @font-face { font-family: 'FontAwesome'; src: url('./assets/fa-solid-900.ttf') format('truetype'), url('./assets/fa-solid-900.woff2') format('woff2'); diff --git a/frontend/src/components/WidgetH2.js b/frontend/src/components/WidgetH2.js new file mode 100644 index 0000000000..c5a25bd391 --- /dev/null +++ b/frontend/src/components/WidgetH2.js @@ -0,0 +1,19 @@ +import React from 'react'; +import PropTypes from 'prop-types'; + +export default function WidgetH2({ children, classNames }) { + return ( +

+ {children} +

+ ); +} + +WidgetH2.propTypes = { + children: PropTypes.node.isRequired, + classNames: PropTypes.string, +}; + +WidgetH2.defaultProps = { + classNames: '', +}; diff --git a/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js b/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js index 1297086797..cfd6362131 100644 --- a/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js +++ b/frontend/src/pages/RegionalDashboard/components/TrainingReportDashboard.js @@ -4,6 +4,7 @@ import { Grid, GridContainer } from '@trussworks/react-uswds'; import Overview from '../../../widgets/TrainingReportDashboardOverview'; import TRReasonList from '../../../widgets/TRReasonList'; import TRHoursOfTrainingByNationalCenter from '../../../widgets/TRHoursOfTrainingByNationalCenter'; +import VTopicFrequency from '../../../widgets/VTopicFrequency'; export default function TrainingReportDashboard() { return ( @@ -36,8 +37,13 @@ export default function TrainingReportDashboard() { /> - - + + + ); diff --git a/frontend/src/pages/RegionalDashboard/index.css b/frontend/src/pages/RegionalDashboard/index.css index 5dc6a415f2..d4372c3682 100644 --- a/frontend/src/pages/RegionalDashboard/index.css +++ b/frontend/src/pages/RegionalDashboard/index.css @@ -17,5 +17,5 @@ } .ttahub--dashboard-widget-heading { - font-size: 1.25em; + font-size: 1.5em; } diff --git a/frontend/src/widgets/BarGraph.js b/frontend/src/widgets/BarGraph.js index f224e22d21..56819d338c 100644 --- a/frontend/src/widgets/BarGraph.js +++ b/frontend/src/widgets/BarGraph.js @@ -101,9 +101,7 @@ function BarGraph({ data }) { return ( <>
- {/* eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex */} -
- Use the arrow keys to scroll graph +
-

{title}

+ + {title} +
diff --git a/frontend/src/widgets/TopicFrequencyGraph.js b/frontend/src/widgets/TopicFrequencyGraph.js index e667fea264..0f7e943e2e 100644 --- a/frontend/src/widgets/TopicFrequencyGraph.js +++ b/frontend/src/widgets/TopicFrequencyGraph.js @@ -26,37 +26,10 @@ export function sortData(data, order, tabular = false) { } } -/** - * - * Takes a string, a reason (or topic, if you prefer) - * provided for an activity report and intersperses it with line breaks - * depending on the length - * - * @param {string} topic - * @returns string with line breaks - */ -export function topicsWithLineBreaks(reason) { - const arrayOfTopics = reason.split(' '); - - return arrayOfTopics.reduce((accumulator, currentValue) => { - const lineBreaks = accumulator.match(/
/g); - const allowedLength = lineBreaks ? lineBreaks.length * 6 : 6; - - // we don't want slashes on their own lines - if (currentValue === '/') { - return `${accumulator} ${currentValue}`; - } - - if (accumulator.length > allowedLength) { - return `${accumulator}
${currentValue}`; - } - - return `${accumulator} ${currentValue}`; - }, ''); -} - export function TopicFrequencyGraphWidget({ - data, loading, + data, + loading, + title, }) { // whether to show the data as accessible widget data or not const [showAccessibleData, setShowAccessibleData] = useState(false); @@ -178,7 +151,7 @@ export function TopicFrequencyGraphWidget({ -

Number of Activity Reports by Topic

+

{title}

 {showAccessibleData ? 'Display graph' : 'Display table'} @@ -232,7 +205,7 @@ export function TopicFrequencyGraphWidget({ { showAccessibleData - ? + ? : (
) } @@ -251,10 +224,11 @@ TopicFrequencyGraphWidget.propTypes = { ), PropTypes.shape({}), ]), loading: PropTypes.bool.isRequired, + title: PropTypes.string, }; TopicFrequencyGraphWidget.defaultProps = { - + title: 'Number of Activity Reports by Topic', data: [], }; diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js index 6c0ff00998..638fa01aea 100644 --- a/frontend/src/widgets/VBarGraph.js +++ b/frontend/src/widgets/VBarGraph.js @@ -3,11 +3,13 @@ import PropTypes from 'prop-types'; import { Grid } from '@trussworks/react-uswds'; // https://github.com/plotly/react-plotly.js/issues/135#issuecomment-501398125 import Plotly from 'plotly.js-basic-dist'; +import { useMediaQuery } from 'react-responsive'; import createPlotlyComponent from 'react-plotly.js/factory'; import colors from '../colors'; import Container from '../components/Container'; import AccessibleWidgetData from './AccessibleWidgetData'; import MediaCaptureButton from '../components/MediaCaptureButton'; +import WidgetH2 from '../components/WidgetH2'; const Plot = createPlotlyComponent(Plotly); @@ -28,6 +30,9 @@ function VBarGraph({ updateShowAccessibleData((current) => !current); } + const isMedium = useMediaQuery({ maxWidth: 1590 }); + const isMobile = useMediaQuery({ maxWidth: 700 }); + useEffect(() => { if (!data || !Array.isArray(data)) { return; @@ -51,9 +56,16 @@ function VBarGraph({ }, }; + const width = (() => { + if (isMobile) return 400; + if (isMedium) return 500; + return 700; + })(); + const layout = { bargap: 0.5, - height: 300, + height: 350, + width, hoverlabel: { bgcolor: '#000', bordercolor: '#000', @@ -72,12 +84,16 @@ function VBarGraph({ }, xaxis: { automargin: true, - fixedrange: true, + autorange: true, tickangle: 0, + title: xAxisLabel, + standoff: 20, }, yaxis: { tickformat: ',.0d', - fixedrange: true, + autorange: true, + title: yAxisLabel, + standoff: 50, }, hovermode: 'none', }; @@ -89,33 +105,36 @@ function VBarGraph({ responsive: true, displayModeBar: false, hovermode: 'none', }, }); - }, [data]); + }, [data, isMedium, isMobile, xAxisLabel, yAxisLabel]); return ( - - -

- {title} -

-

{subtitle}

- - + +
+
+ + {title} + +

{subtitle}

+
+ {!showAccessibleData + ? ( + + ) + : null} - - +
+ { showAccessibleData ? ( @@ -127,11 +146,7 @@ function VBarGraph({ ) : ( <> -
-
- { yAxisLabel } -
- +
-
- { xAxisLabel } -
)} diff --git a/frontend/src/widgets/VTopicFrequency.js b/frontend/src/widgets/VTopicFrequency.js new file mode 100644 index 0000000000..37c43bed20 --- /dev/null +++ b/frontend/src/widgets/VTopicFrequency.js @@ -0,0 +1,4 @@ +import { TopicFrequencyGraphWidget } from './TopicFrequencyGraph'; +import withWidgetData from './withWidgetData'; + +export default withWidgetData(TopicFrequencyGraphWidget, 'trSessionsByTopic'); diff --git a/frontend/src/widgets/__tests__/TopicFrequencyGraph.js b/frontend/src/widgets/__tests__/TopicFrequencyGraph.js index 693f8c616a..9fe92dde71 100644 --- a/frontend/src/widgets/__tests__/TopicFrequencyGraph.js +++ b/frontend/src/widgets/__tests__/TopicFrequencyGraph.js @@ -9,7 +9,6 @@ import { import userEvent from '@testing-library/user-event'; import { TopicFrequencyGraphWidget, - topicsWithLineBreaks, sortData, SORT_ORDER, } from '../TopicFrequencyGraph'; @@ -129,11 +128,6 @@ describe('Topic & Frequency Graph Widget', () => { expect(await screen.findByText('Loading')).toBeInTheDocument(); }); - it('correctly inserts line breaks', () => { - const formattedtopic = topicsWithLineBreaks('Equity, Culture & Language'); - expect(formattedtopic).toBe(' Equity,
Culture
&
Language'); - }); - it('the sort control works', async () => { renderArGraphOverview({ data: [...TEST_DATA] }); const button = screen.getByRole('button', { name: /change topic graph order/i }); @@ -155,7 +149,7 @@ describe('Topic & Frequency Graph Widget', () => { it('handles switching display contexts', async () => { renderArGraphOverview({ data: [...TEST_DATA] }); - const button = await screen.findByRole('button', { name: 'display number of activity reports by topic data as table' }); + const button = await screen.findByRole('button', { name: /display Number of Activity Reports by Topic as table/i }); act(() => userEvent.click(button)); const firstRowHeader = await screen.findByRole('cell', { @@ -166,7 +160,7 @@ describe('Topic & Frequency Graph Widget', () => { const firstTableCell = await screen.findByRole('cell', { name: /155/i }); expect(firstTableCell).toBeInTheDocument(); - const viewGraph = await screen.findByRole('button', { name: 'display number of activity reports by topic data as graph' }); + const viewGraph = await screen.findByRole('button', { name: /display Number of Activity Reports by Topic as graph/i }); act(() => userEvent.click(viewGraph)); expect(firstRowHeader).not.toBeInTheDocument(); diff --git a/frontend/src/widgets/widgets.scss b/frontend/src/widgets/widgets.scss new file mode 100644 index 0000000000..0b912a9054 --- /dev/null +++ b/frontend/src/widgets/widgets.scss @@ -0,0 +1,17 @@ +.ttahub-widget-heading-grid { + align-items: start; + display: grid; + grid-template-columns: 1fr; + gap: 1em; + width: 100%; +} + +@media(min-width: 1300px) { + .ttahub-widget-heading-grid { + grid-template-columns: repeat(2, 1fr) repeat(2, max-content); + } + + .ttahub-widget-heading-grid--title, .tta-widget-heading-grid--actions { + grid-column: span 2; + } +} \ No newline at end of file From 088bd291c6e8c436c0029ce1adc782440e9e5afb Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 13:13:30 -0400 Subject: [PATCH 52/75] Update backend to match FE graph expectation --- src/widgets/trSessionsByTopic.ts | 6 +++--- src/widgets/trSessionsByTopics.test.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/widgets/trSessionsByTopic.ts b/src/widgets/trSessionsByTopic.ts index 4c5e815a31..68cb243bf3 100644 --- a/src/widgets/trSessionsByTopic.ts +++ b/src/widgets/trSessionsByTopic.ts @@ -37,9 +37,9 @@ export default async function trSessionByTopic( ]; const dataStruct = topics.map((topic: { name: string }) => ({ - name: topic.name, + topic: topic.name, count: 0, - })) as { name: string, count: number }[]; + })) as { topic: string, count: number }[]; const response = reports.reduce((acc, report) => { const { sessionReports } = report; @@ -47,7 +47,7 @@ export default async function trSessionByTopic( const { objectiveTopics } = sessionReport.data; objectiveTopics.forEach((topic) => { - const d = dataStruct.find((c) => c.name === topic); + const d = dataStruct.find((c) => c.topic === topic); if (d) { d.count += 1; } diff --git a/src/widgets/trSessionsByTopics.test.js b/src/widgets/trSessionsByTopics.test.js index 328656a08b..25a92d2a67 100644 --- a/src/widgets/trSessionsByTopics.test.js +++ b/src/widgets/trSessionsByTopics.test.js @@ -287,10 +287,10 @@ describe('TR sessions by topic', () => { // run our function const data = await trSessionsByTopic(scopes); - const firstTopic = data.find((d) => topic1.name === d.name); + const firstTopic = data.find((d) => topic1.name === d.topic); expect(firstTopic.count).toBe(1); - const secondTopic = data.find((d) => topic2.name === d.name); + const secondTopic = data.find((d) => topic2.name === d.topic); expect(secondTopic.count).toBe(1); }); }); From 72ba8d3e4b0059bb8dc5cb92af15095a85996c99 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 13:37:34 -0400 Subject: [PATCH 53/75] Tests and fixes --- .../__tests__/TrainingReportDashboard.js | 51 ++++++++++++++++ frontend/src/widgets/VBarGraph.js | 11 ++-- frontend/src/widgets/__tests__/BarGraph.js | 1 - frontend/src/widgets/__tests__/VBarGraph.js | 61 +++++++++++++++++++ 4 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 frontend/src/pages/RegionalDashboard/components/__tests__/TrainingReportDashboard.js create mode 100644 frontend/src/widgets/__tests__/VBarGraph.js diff --git a/frontend/src/pages/RegionalDashboard/components/__tests__/TrainingReportDashboard.js b/frontend/src/pages/RegionalDashboard/components/__tests__/TrainingReportDashboard.js new file mode 100644 index 0000000000..d508ff0395 --- /dev/null +++ b/frontend/src/pages/RegionalDashboard/components/__tests__/TrainingReportDashboard.js @@ -0,0 +1,51 @@ +import '@testing-library/jest-dom'; +import React from 'react'; +import { + render, screen, +} from '@testing-library/react'; +import fetchMock from 'fetch-mock'; +import TrainingReportDashboard from '../TrainingReportDashboard'; + +describe('Training report Dashboard page', () => { + const hoursOfTrainingUrl = '/api/widgets/trHoursOfTrainingByNationalCenter'; + const reasonListUrl = '/api/widgets/trReasonList'; + const overviewUrl = '/api/widgets/trOverview'; + const sessionsByTopicUrl = '/api/widgets/trSessionsByTopic'; + + beforeEach(async () => { + fetchMock.get(overviewUrl, { + numReports: '0', + totalRecipients: '0', + recipientPercentage: '0%', + numGrants: '0', + numRecipients: '0', + sumDuration: '0', + numParticipants: '0', + numSessions: '0', + }); + fetchMock.get(reasonListUrl, []); + fetchMock.get(hoursOfTrainingUrl, []); + fetchMock.get(sessionsByTopicUrl, []); + }); + + afterEach(() => fetchMock.restore()); + + const renderTest = () => { + render(); + }; + + it('renders and fetches data', async () => { + renderTest(); + + expect(fetchMock.calls(overviewUrl)).toHaveLength(1); + expect(fetchMock.calls(reasonListUrl)).toHaveLength(1); + expect(fetchMock.calls(hoursOfTrainingUrl)).toHaveLength(1); + expect(fetchMock.calls(sessionsByTopicUrl)).toHaveLength(1); + + expect(document.querySelector('.smart-hub--dashboard-overview')).toBeTruthy(); + + expect(screen.getByText('Reasons in Training Reports')).toBeInTheDocument(); + expect(screen.getByText('Hours of training by National Center')).toBeInTheDocument(); + expect(screen.getByText('Number of TR sessions by topic')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js index 638fa01aea..bfba23aa91 100644 --- a/frontend/src/widgets/VBarGraph.js +++ b/frontend/src/widgets/VBarGraph.js @@ -107,8 +107,10 @@ function VBarGraph({ }); }, [data, isMedium, isMobile, xAxisLabel, yAxisLabel]); + const tableData = data.map((row) => ({ data: [row.name, row.count] })); + return ( - +
@@ -130,6 +132,7 @@ function VBarGraph({ type="button" className="usa-button usa-button--unstyled" onClick={toggleAccessibleData} + aria-label={showAccessibleData ? `Display ${title} as graph` : `Display ${title} as table`} > {showAccessibleData ? 'Display graph' : 'Display table'} @@ -139,9 +142,9 @@ function VBarGraph({ { showAccessibleData ? ( ) : ( diff --git a/frontend/src/widgets/__tests__/BarGraph.js b/frontend/src/widgets/__tests__/BarGraph.js index 1977330f66..82d7712ce0 100644 --- a/frontend/src/widgets/__tests__/BarGraph.js +++ b/frontend/src/widgets/__tests__/BarGraph.js @@ -1,4 +1,3 @@ -/* eslint-disable jest/no-disabled-tests */ import '@testing-library/jest-dom'; import React from 'react'; import { diff --git a/frontend/src/widgets/__tests__/VBarGraph.js b/frontend/src/widgets/__tests__/VBarGraph.js new file mode 100644 index 0000000000..c6039b8c55 --- /dev/null +++ b/frontend/src/widgets/__tests__/VBarGraph.js @@ -0,0 +1,61 @@ +import '@testing-library/jest-dom'; +import React from 'react'; +import { + render, + waitFor, + act, + screen, +} from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import VBarGraph from '../VBarGraph'; + +const TEST_DATA = [{ + name: 'one', + count: 1, +}, +{ + name: 'two / two and a half', + count: 2, +}, +{ + name: 'three is the number than comes after two and with that we think about it', + count: 0, +}]; + +const renderBarGraph = async () => { + act(() => { + render(); + }); +}; + +describe('VBar Graph', () => { + it('is shown', async () => { + renderBarGraph(); + + await waitFor(() => expect(document.querySelector('svg')).not.toBe(null)); + + const point1 = document.querySelector('g.ytick'); + // eslint-disable-next-line no-underscore-dangle + expect(point1.__data__.text).toBe('0'); + + const point2 = document.querySelector('g.xtick'); + // eslint-disable-next-line no-underscore-dangle + expect(point2.__data__.text).toBe('one'); + }); + + it('toggles table view', async () => { + act(() => { + renderBarGraph(); + }); + + await waitFor(() => expect(document.querySelector('svg')).not.toBe(null)); + + const button = await screen.findByRole('button', { name: /as table/i }); + act(() => { + userEvent.click(button); + }); + + const table = document.querySelector('table'); + expect(table).not.toBeNull(); + }); +}); From 946755d67f664711c6e595599841b54c8ada8ed3 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 13:44:26 -0400 Subject: [PATCH 54/75] Regional dashboard test e2e test update --- tests/e2e/regional-dashboard.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/regional-dashboard.spec.ts b/tests/e2e/regional-dashboard.spec.ts index e7a3e13e90..b5c8b6d63e 100644 --- a/tests/e2e/regional-dashboard.spec.ts +++ b/tests/e2e/regional-dashboard.spec.ts @@ -48,7 +48,7 @@ test('Regional Dashboard', async ({ page }) => { ]); // view the topics as a table - await page.getByRole('button', { name: 'display number of activity reports by topic data as table' }).click(); + await page.getByRole('button', { name: /display Number of Activity Reports by Topic as table/i }).click(); // change the topics graph order await page.getByRole('button', { name: 'toggle Change topic graph order menu' }).click(); From 0a79161838b61dbdffb8390b3a18215a10acf956 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Wed, 10 Apr 2024 14:11:27 -0400 Subject: [PATCH 55/75] Axe updates --- frontend/src/widgets/BarGraph.js | 4 +++- tests/e2e/axe.spec.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/widgets/BarGraph.js b/frontend/src/widgets/BarGraph.js index 56819d338c..f224e22d21 100644 --- a/frontend/src/widgets/BarGraph.js +++ b/frontend/src/widgets/BarGraph.js @@ -101,7 +101,9 @@ function BarGraph({ data }) { return ( <>
-
+ {/* eslint-disable-next-line jsx-a11y/no-noninteractive-tabindex */} +
+ Use the arrow keys to scroll graph Date: Wed, 10 Apr 2024 14:45:29 -0400 Subject: [PATCH 56/75] add back test per Matt --- src/lib/cache.test.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/cache.test.js b/src/lib/cache.test.js index 8e17c93edb..ac720d3e2c 100644 --- a/src/lib/cache.test.js +++ b/src/lib/cache.test.js @@ -1,4 +1,3 @@ -/* eslint-disable jest/no-commented-out-tests */ import { createClient } from 'redis'; import getCachedResponse from './cache'; @@ -22,7 +21,6 @@ describe('getCachedResponse', () => { process.env = ORIGINAL_ENV; // restore original env }); - /* it('returns the cached response', async () => { const callback = jest.fn(() => 'new value'); createClient.mockImplementation(() => ({ @@ -34,7 +32,6 @@ describe('getCachedResponse', () => { const response = await getCachedResponse('key', callback); expect(response).toEqual('value'); }); - */ it('calls the callback when there is no cached response', async () => { createClient.mockImplementation(() => ({ From 262fa83b639d6980c2c73e51163e8660bc674034 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:47:49 +0000 Subject: [PATCH 57/75] Bump protobufjs from 7.2.4 to 7.2.6 Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.2.4 to 7.2.6. - [Release notes](https://github.com/protobufjs/protobuf.js/releases) - [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md) - [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.2.4...protobufjs-v7.2.6) --- updated-dependencies: - dependency-name: protobufjs dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4941dc3d4f..7314ca851b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10964,9 +10964,9 @@ proto-list@~1.2.1: integrity sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA== protobufjs@^7.2.4: - version "7.2.4" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.4.tgz#3fc1ec0cdc89dd91aef9ba6037ba07408485c3ae" - integrity sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ== + version "7.2.6" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.2.6.tgz#4a0ccd79eb292717aacf07530a07e0ed20278215" + integrity sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" From 09260b299c5a2d4882e3d6e8e3b52aa28cd9150e Mon Sep 17 00:00:00 2001 From: Adam Levin Date: Thu, 11 Apr 2024 10:03:02 -0400 Subject: [PATCH 58/75] fix issue Matt found with sorting --- frontend/src/widgets/HorizontalTableWidget.js | 1 - .../widgets/ResourcesAssociatedWithTopics.js | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/src/widgets/HorizontalTableWidget.js b/frontend/src/widgets/HorizontalTableWidget.js index 12ae204226..6c8e3fc711 100644 --- a/frontend/src/widgets/HorizontalTableWidget.js +++ b/frontend/src/widgets/HorizontalTableWidget.js @@ -49,7 +49,6 @@ export default function HorizontalTableWidget( onClick={() => { requestSort(name); }} - onKeyDown={() => requestSort(name)} className={`usa-button usa-button--unstyled sortable ${sortClassName}`} aria-label={`${displayName}. Activate to sort ${sortClassName === 'asc' ? 'descending' : 'ascending' }`} diff --git a/frontend/src/widgets/ResourcesAssociatedWithTopics.js b/frontend/src/widgets/ResourcesAssociatedWithTopics.js index eb4df4df41..08a1954f71 100644 --- a/frontend/src/widgets/ResourcesAssociatedWithTopics.js +++ b/frontend/src/widgets/ResourcesAssociatedWithTopics.js @@ -95,15 +95,16 @@ function ResourcesAssociatedWithTopics({ // Value sort. const sortValueA = direction === 'asc' ? 1 : -1; - const sortValueB = direction === 'asc' ? -1 : -1; - valuesToSort.sort( - (a, b) => ( - // eslint-disable-next-line no-nested-ternary - (a.sortBy > b.sortBy) ? sortValueA - : ((b.sortBy > a.sortBy) - ? sortValueB : 0) - ), - ); + const sortValueB = direction === 'asc' ? -1 : 1; + valuesToSort.sort((a, b) => { + if (a.sortBy > b.sortBy) { + return sortValueA; + } if (b.sortBy > a.sortBy) { + return sortValueB; + } + return 0; + }); + setTopicUse(valuesToSort); setOffset(0); setSortConfig({ sortBy, direction, activePage: 1 }); From 164eaa1e7b1a0815e71949e44457295a21565b57 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 12:01:59 -0400 Subject: [PATCH 59/75] Deploy to sandbox --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0071206c46..33806dc7ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -291,7 +291,7 @@ parameters: default: "mb/TTAHUB-2501/front-end-goal-name-filter" type: string sandbox_git_branch: # change to feature branch to test deployment - default: "mb/TTAHUB-2510/add-IST-visit-dropdown" + default: "mb/TTAHUB/frontend-for-tr-dashboard" type: string prod_new_relic_app_id: default: "877570491" From d2f899445208c33f126c2662bed1de93f00da955 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:01:11 -0400 Subject: [PATCH 60/75] Test image requested --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0071206c46..7ce200dd40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: large + resource_class: arm.large deploy: executor: docker-executor steps: From 4f64ea7440d1e318b05f3440fb048334fc3a9baf Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:03:02 -0400 Subject: [PATCH 61/75] Take patricks advice instead --- .circleci/config.yml | 2 +- docker-compose.override.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce200dd40..0071206c46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: arm.large + resource_class: large deploy: executor: docker-executor steps: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index fbfe4d4da6..acc8628f1f 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -61,7 +61,7 @@ services: - ".:/app:rw" owasp_zap_backend: image: owasp/zap2docker-stable:latest - platform: linux/arm64 + # platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html volumes: @@ -71,7 +71,7 @@ services: - backend owasp_zap_similarity: image: owasp/zap2docker-stable:latest - platform: linux/arm64 + # platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html volumes: From db9429a3147bad3d250e4c561dd98ea45e8a58c4 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:13:12 -0400 Subject: [PATCH 62/75] Try this instead --- .circleci/config.yml | 2 +- docker-compose.override.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0071206c46..7ce200dd40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: large + resource_class: arm.large deploy: executor: docker-executor steps: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index acc8628f1f..fbfe4d4da6 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -61,7 +61,7 @@ services: - ".:/app:rw" owasp_zap_backend: image: owasp/zap2docker-stable:latest - # platform: linux/arm64 + platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html volumes: @@ -71,7 +71,7 @@ services: - backend owasp_zap_similarity: image: owasp/zap2docker-stable:latest - # platform: linux/arm64 + platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html volumes: From 8ffdcd6b8ce0bbfe39a624d73575b52b017ba0f5 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:31:27 -0400 Subject: [PATCH 63/75] Use older image --- .circleci/config.yml | 2 +- docker-compose.override.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce200dd40..0071206c46 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: arm.large + resource_class: large deploy: executor: docker-executor steps: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index fbfe4d4da6..80b1f7f6e8 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -60,7 +60,7 @@ services: volumes: - ".:/app:rw" owasp_zap_backend: - image: owasp/zap2docker-stable:latest + image: ghcr.io/zaproxy/zap-archives/zap2docker-stable:2.14.0 platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html @@ -70,7 +70,7 @@ services: depends_on: - backend owasp_zap_similarity: - image: owasp/zap2docker-stable:latest + image: ghcr.io/zaproxy/zap-archives/zap2docker-stable:2.14.0 platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html From 03c8a5d3780ffced576d6d42ddd4b2a22b9bdb75 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:33:20 -0400 Subject: [PATCH 64/75] Try a diff image --- docker-compose.override.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 80b1f7f6e8..96a13761da 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -60,7 +60,7 @@ services: volumes: - ".:/app:rw" owasp_zap_backend: - image: ghcr.io/zaproxy/zap-archives/zap2docker-stable:2.14.0 + image: softwaresecurityproject/zap-bare:latest platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html @@ -70,7 +70,7 @@ services: depends_on: - backend owasp_zap_similarity: - image: ghcr.io/zaproxy/zap-archives/zap2docker-stable:2.14.0 + image: softwaresecurityproject/zap-bare:latest platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html From b965323d4d53ffaf9c5f7197fa8c2f0dc1c4ae2c Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:49:19 -0400 Subject: [PATCH 65/75] add resize observer hook --- frontend/package.json | 1 + frontend/src/hooks/useSize.js | 17 +++++++++++++++++ frontend/src/widgets/VBarGraph.js | 17 +++++------------ frontend/yarn.lock | 24 ++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 frontend/src/hooks/useSize.js diff --git a/frontend/package.json b/frontend/package.json index d0837972db..79ce687a4c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "@fortawesome/free-solid-svg-icons": "^6.4.2", "@fortawesome/react-fontawesome": "^0.2.0", "@hookform/error-message": "^0.0.5", + "@react-hook/resize-observer": "^1.2.6", "@trussworks/react-uswds": "4.1.1", "@ttahub/common": "2.0.18", "@use-it/interval": "^1.0.0", diff --git a/frontend/src/hooks/useSize.js b/frontend/src/hooks/useSize.js new file mode 100644 index 0000000000..09521708da --- /dev/null +++ b/frontend/src/hooks/useSize.js @@ -0,0 +1,17 @@ +// https://www.npmjs.com/package/@react-hook/resize-observer +import { useState, useLayoutEffect } from 'react'; +import useResizeObserver from '@react-hook/resize-observer'; + +const useSize = (target) => { + const [size, setSize] = useState(); + + useLayoutEffect(() => { + setSize(target.current.getBoundingClientRect()); + }, [target]); + + // Where the magic happens + useResizeObserver(target, (entry) => setSize(entry.contentRect)); + return size; +}; + +export default useSize; diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js index bfba23aa91..8cb5e88514 100644 --- a/frontend/src/widgets/VBarGraph.js +++ b/frontend/src/widgets/VBarGraph.js @@ -3,13 +3,13 @@ import PropTypes from 'prop-types'; import { Grid } from '@trussworks/react-uswds'; // https://github.com/plotly/react-plotly.js/issues/135#issuecomment-501398125 import Plotly from 'plotly.js-basic-dist'; -import { useMediaQuery } from 'react-responsive'; import createPlotlyComponent from 'react-plotly.js/factory'; import colors from '../colors'; import Container from '../components/Container'; import AccessibleWidgetData from './AccessibleWidgetData'; import MediaCaptureButton from '../components/MediaCaptureButton'; import WidgetH2 from '../components/WidgetH2'; +import useSize from '../hooks/useSize'; const Plot = createPlotlyComponent(Plotly); @@ -30,11 +30,10 @@ function VBarGraph({ updateShowAccessibleData((current) => !current); } - const isMedium = useMediaQuery({ maxWidth: 1590 }); - const isMobile = useMediaQuery({ maxWidth: 700 }); + const size = useSize(bars); useEffect(() => { - if (!data || !Array.isArray(data)) { + if (!data || !Array.isArray(data) || !size) { return; } @@ -56,16 +55,10 @@ function VBarGraph({ }, }; - const width = (() => { - if (isMobile) return 400; - if (isMedium) return 500; - return 700; - })(); - const layout = { bargap: 0.5, height: 350, - width, + width: size.width - 40, hoverlabel: { bgcolor: '#000', bordercolor: '#000', @@ -105,7 +98,7 @@ function VBarGraph({ responsive: true, displayModeBar: false, hovermode: 'none', }, }); - }, [data, isMedium, isMobile, xAxisLabel, yAxisLabel]); + }, [data, xAxisLabel, size, yAxisLabel]); const tableData = data.map((row) => ({ data: [row.name, row.count] })); diff --git a/frontend/yarn.lock b/frontend/yarn.lock index c155d962dc..c4fb6940b8 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1844,6 +1844,11 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" +"@juggle/resize-observer@^3.3.1": + version "3.4.0" + resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.4.0.tgz#08d6c5e20cf7e4cc02fd181c4b0c225cd31dbb60" + integrity sha512-dfLbk+PwWvFzSxwk3n5ySL0hfBog779o8h68wK/7/APo/7cgyWp5jcXockbxdk5kFRkbeXWm4Fbi9FrdN381sA== + "@leichtgewicht/ip-codec@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" @@ -2053,6 +2058,25 @@ schema-utils "^3.0.0" source-map "^0.7.3" +"@react-hook/latest@^1.0.2": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@react-hook/latest/-/latest-1.0.3.tgz#c2d1d0b0af8b69ec6e2b3a2412ba0768ac82db80" + integrity sha512-dy6duzl+JnAZcDbNTfmaP3xHiKtbXYOaz3G51MGVljh548Y8MWzTr+PHLOfvpypEVW9zwvl+VyKjbWKEVbV1Rg== + +"@react-hook/passive-layout-effect@^1.2.0": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@react-hook/passive-layout-effect/-/passive-layout-effect-1.2.1.tgz#c06dac2d011f36d61259aa1c6df4f0d5e28bc55e" + integrity sha512-IwEphTD75liO8g+6taS+4oqz+nnroocNfWVHWz7j+N+ZO2vYrc6PV1q7GQhuahL0IOR7JccFTsFKQ/mb6iZWAg== + +"@react-hook/resize-observer@^1.2.6": + version "1.2.6" + resolved "https://registry.yarnpkg.com/@react-hook/resize-observer/-/resize-observer-1.2.6.tgz#9a8cf4c5abb09becd60d1d65f6bf10eec211e291" + integrity sha512-DlBXtLSW0DqYYTW3Ft1/GQFZlTdKY5VAFIC4+km6IK5NiPPDFchGbEJm1j6pSgMqPRHbUQgHJX7RaR76ic1LWA== + dependencies: + "@juggle/resize-observer" "^3.3.1" + "@react-hook/latest" "^1.0.2" + "@react-hook/passive-layout-effect" "^1.2.0" + "@redux-saga/core@^1.2.2": version "1.2.2" resolved "https://registry.yarnpkg.com/@redux-saga/core/-/core-1.2.2.tgz#99b1daac93a42feecd9bab449f452f56f3155fea" From 3c4088a14306a2d9b3f7740f1b55a575da2b13b1 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:50:37 -0400 Subject: [PATCH 66/75] Update image again --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0071206c46..7ce200dd40 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: large + resource_class: arm.large deploy: executor: docker-executor steps: From 441133fa1c897bac9eabcdcd7be28965d9592fa3 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:55:41 -0400 Subject: [PATCH 67/75] address height issue --- frontend/src/widgets/AccessibleWidgetData.js | 2 +- frontend/src/widgets/VBarGraph.css | 3 +++ frontend/src/widgets/VBarGraph.js | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 frontend/src/widgets/VBarGraph.css diff --git a/frontend/src/widgets/AccessibleWidgetData.js b/frontend/src/widgets/AccessibleWidgetData.js index f7f4931f90..97c533a367 100644 --- a/frontend/src/widgets/AccessibleWidgetData.js +++ b/frontend/src/widgets/AccessibleWidgetData.js @@ -14,7 +14,7 @@ export default function AccessibleWidgetData({ caption, columnHeadings, rows }) } return ( -
+
{caption} diff --git a/frontend/src/widgets/VBarGraph.css b/frontend/src/widgets/VBarGraph.css new file mode 100644 index 0000000000..442958cadf --- /dev/null +++ b/frontend/src/widgets/VBarGraph.css @@ -0,0 +1,3 @@ +.smarthub-vbar-graph { + height: calc(100% - 1.5em); +} \ No newline at end of file diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js index 8cb5e88514..7b14d79e7f 100644 --- a/frontend/src/widgets/VBarGraph.js +++ b/frontend/src/widgets/VBarGraph.js @@ -10,6 +10,7 @@ import AccessibleWidgetData from './AccessibleWidgetData'; import MediaCaptureButton from '../components/MediaCaptureButton'; import WidgetH2 from '../components/WidgetH2'; import useSize from '../hooks/useSize'; +import './VBarGraph.css'; const Plot = createPlotlyComponent(Plotly); From e5d3c9912242beec48dff6be388250c905ee809a Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 13:56:11 -0400 Subject: [PATCH 68/75] Revert change to accessible table widget --- frontend/src/widgets/AccessibleWidgetData.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/widgets/AccessibleWidgetData.js b/frontend/src/widgets/AccessibleWidgetData.js index 97c533a367..f7f4931f90 100644 --- a/frontend/src/widgets/AccessibleWidgetData.js +++ b/frontend/src/widgets/AccessibleWidgetData.js @@ -14,7 +14,7 @@ export default function AccessibleWidgetData({ caption, columnHeadings, rows }) } return ( -
+
{caption} From 7f48a87120136bab1f7fa9f06b717491e498d2fe Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 14:07:45 -0400 Subject: [PATCH 69/75] Retry --- .circleci/config.yml | 4 ++-- docker-compose.override.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ce200dd40..c5cc0cb63e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -644,7 +644,7 @@ jobs: command: ./bin/ping-server 8080 - run: name: Pull OWASP ZAP docker image - command: docker pull owasp/zap2docker-stable:latest + command: docker pull docker pull owasp/zap2docker-bare:latest - run: name: Make reports directory group writeable command: chmod g+w reports @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: arm.large + resource_class: large deploy: executor: docker-executor steps: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 96a13761da..2813595de3 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -60,8 +60,8 @@ services: volumes: - ".:/app:rw" owasp_zap_backend: - image: softwaresecurityproject/zap-bare:latest - platform: linux/arm64 + image: owasp/zap2docker-bare:latest + # platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html volumes: @@ -70,8 +70,8 @@ services: depends_on: - backend owasp_zap_similarity: - image: softwaresecurityproject/zap-bare:latest - platform: linux/arm64 + image: owasp/zap2docker-bare:latest + # platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html volumes: From 68bd801f6fed5ebe6a0e3d9110eb94ac9a16fa11 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 14:09:12 -0400 Subject: [PATCH 70/75] Remove double docker pull --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c5cc0cb63e..3399a9c0bb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -644,7 +644,7 @@ jobs: command: ./bin/ping-server 8080 - run: name: Pull OWASP ZAP docker image - command: docker pull docker pull owasp/zap2docker-bare:latest + command: docker pull owasp/zap2docker-bare:latest - run: name: Make reports directory group writeable command: chmod g+w reports From 1bea9e95a9870ea6eec9943eca350948ea4e1b10 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 14:20:36 -0400 Subject: [PATCH 71/75] New docker image, trying again --- .circleci/config.yml | 2 +- docker-compose.override.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3399a9c0bb..65cab9ad5e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -644,7 +644,7 @@ jobs: command: ./bin/ping-server 8080 - run: name: Pull OWASP ZAP docker image - command: docker pull owasp/zap2docker-bare:latest + command: docker pull softwaresecurityproject/zap-stable:latest - run: name: Make reports directory group writeable command: chmod g+w reports diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 2813595de3..a671b13f5c 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -60,7 +60,7 @@ services: volumes: - ".:/app:rw" owasp_zap_backend: - image: owasp/zap2docker-bare:latest + image: softwaresecurityproject/zap-stable:latest # platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html @@ -70,7 +70,7 @@ services: depends_on: - backend owasp_zap_similarity: - image: owasp/zap2docker-bare:latest + image: softwaresecurityproject/zap-stable:latest # platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html From 83f8b9194c5292374bd296732626c696a4a4c393 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 14:29:26 -0400 Subject: [PATCH 72/75] Update CI machine --- .circleci/config.yml | 2 +- docker-compose.override.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 65cab9ad5e..b5595ab4c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -653,7 +653,7 @@ jobs: command: ./bin/run-owasp-scan - store_artifacts: path: reports/owasp_report.html - resource_class: large + resource_class: arm.large deploy: executor: docker-executor steps: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index a671b13f5c..2c8bd0e9bd 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -61,7 +61,7 @@ services: - ".:/app:rw" owasp_zap_backend: image: softwaresecurityproject/zap-stable:latest - # platform: linux/arm64 + platform: linux/arm64 user: zap command: zap-full-scan.py -t http://backend:8080 -c zap.conf -i -r owasp_report.html volumes: @@ -71,7 +71,7 @@ services: - backend owasp_zap_similarity: image: softwaresecurityproject/zap-stable:latest - # platform: linux/arm64 + platform: linux/arm64 user: zap command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html volumes: From ff803ec9fe95046e60761d294fe4ab44983a1e2b Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Thu, 11 Apr 2024 14:40:40 -0400 Subject: [PATCH 73/75] Update image ref in sh script --- bin/run-owasp-scan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-owasp-scan b/bin/run-owasp-scan index 2e36a8a618..15d076112f 100755 --- a/bin/run-owasp-scan +++ b/bin/run-owasp-scan @@ -24,6 +24,6 @@ docker run \ --rm \ --user zap:$(id -g) \ --network=$network \ - -t owasp/zap2docker-stable:latest zap-baseline.py \ + -t softwaresecurityproject/zap-stable:latest zap-baseline.py \ -t http://server:8080 \ -c zap.conf -I -i -r owasp_report.html From 1eafaef389f2a977964eb4b2b322f49086acabd0 Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Fri, 12 Apr 2024 09:48:53 -0400 Subject: [PATCH 74/75] Update sorting and test for it --- frontend/src/widgets/TopicFrequencyGraph.js | 10 +++- .../widgets/__tests__/TopicFrequencyGraph.js | 47 ++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/frontend/src/widgets/TopicFrequencyGraph.js b/frontend/src/widgets/TopicFrequencyGraph.js index 0f7e943e2e..e67df87c9d 100644 --- a/frontend/src/widgets/TopicFrequencyGraph.js +++ b/frontend/src/widgets/TopicFrequencyGraph.js @@ -15,12 +15,20 @@ export const SORT_ORDER = { }; export function sortData(data, order, tabular = false) { + // if order === SORT_ORDER.ALPHA, sort alphabetically if (order === SORT_ORDER.ALPHA) { data.sort((a, b) => a.topic.localeCompare(b.topic)); } else { - data.sort((a, b) => b.count - a.count); + // sort by count and then alphabetically + data.sort((a, b) => { + if (a.count === b.count) { + return a.topic.localeCompare(b.topic); + } + return b.count - a.count; + }); } + // the orientation is reversed visually in the table if (!tabular) { data.reverse(); } diff --git a/frontend/src/widgets/__tests__/TopicFrequencyGraph.js b/frontend/src/widgets/__tests__/TopicFrequencyGraph.js index 9fe92dde71..48760732a7 100644 --- a/frontend/src/widgets/__tests__/TopicFrequencyGraph.js +++ b/frontend/src/widgets/__tests__/TopicFrequencyGraph.js @@ -53,7 +53,7 @@ describe('Topic & Frequency Graph Widget', () => { }); it('correctly sorts data by count', () => { - const data = [...TEST_DATA]; + let data = [...TEST_DATA]; sortData(data, SORT_ORDER.DESC); expect(data).toStrictEqual([ { @@ -72,15 +72,44 @@ describe('Topic & Frequency Graph Widget', () => { topic: 'CLASS: Instructional Support', count: 12, }, + { + topic: 'Fiscal / Budget', + count: 0, + }, { topic: 'Human Resources', count: 0, }, + ].reverse()); + + data = [...TEST_DATA]; + sortData(data, SORT_ORDER.DESC, true); + expect(data).toStrictEqual([ + { + topic: 'Community and Self-Assessment', + count: 155, + }, + { + topic: 'Family Support Services', + count: 53, + }, + { + topic: 'Five-Year Grant', + count: 33, + }, + { + topic: 'CLASS: Instructional Support', + count: 12, + }, { topic: 'Fiscal / Budget', count: 0, }, - ].reverse()); + { + topic: 'Human Resources', + count: 0, + }, + ]); }); it('correctly sorts data alphabetically', () => { @@ -136,15 +165,21 @@ describe('Topic & Frequency Graph Widget', () => { act(() => userEvent.click(aZ)); const apply = screen.getByRole('button', { name: 'Apply filters for the Change topic graph order menu' }); - const point1 = document.querySelector('g.ytick'); + // this won't change because we sort count and then alphabetically + // and this is always last in that case + const firstPoint = document.querySelector('g.ytick'); + // eslint-disable-next-line no-underscore-dangle + expect(firstPoint.__data__.text).toBe('Human Resources'); + + const point1 = Array.from(document.querySelectorAll('g.ytick')).pop(); // eslint-disable-next-line no-underscore-dangle - expect(point1.__data__.text).toBe('Fiscal / Budget'); + expect(point1.__data__.text).toBe('Community and Self-Assessment'); act(() => userEvent.click(apply)); - const point2 = document.querySelector('g.ytick'); + const point2 = Array.from(document.querySelectorAll('g.ytick')).pop(); // eslint-disable-next-line no-underscore-dangle - expect(point2.__data__.text).toBe('Human Resources'); + expect(point2.__data__.text).toBe('CLASS: Instructional Support'); }); it('handles switching display contexts', async () => { From dac2c813b9d69193ce4c048541dd88ac0dc9c11e Mon Sep 17 00:00:00 2001 From: Matt Bevilacqua Date: Fri, 12 Apr 2024 12:14:54 -0400 Subject: [PATCH 75/75] Design review changes --- bin/ping-server | 2 +- frontend/src/components/MediaCaptureButton.js | 5 +++-- .../src/components/__tests__/MediaCaptureButton.js | 2 +- frontend/src/widgets/DashboardOverview.js | 2 +- frontend/src/widgets/TopicFrequencyGraph.js | 1 + frontend/src/widgets/VBarGraph.js | 12 +++++++++--- .../__tests__/TrainingReportDashboardOverview.js | 4 ++-- 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bin/ping-server b/bin/ping-server index a1d298087b..d84846fe64 100755 --- a/bin/ping-server +++ b/bin/ping-server @@ -13,5 +13,5 @@ until $(curl --output /dev/null --max-time 10 --silent --head --fail http://loca fi attempt_counter=$(($attempt_counter+1)) - sleep 10 + sleep 30 done diff --git a/frontend/src/components/MediaCaptureButton.js b/frontend/src/components/MediaCaptureButton.js index 589677c5fe..0c7aa97070 100644 --- a/frontend/src/components/MediaCaptureButton.js +++ b/frontend/src/components/MediaCaptureButton.js @@ -4,7 +4,7 @@ import html2canvas from 'html2canvas'; import { Button } from '@trussworks/react-uswds'; export default function MediaCaptureButton({ - reference, className, buttonText, id, + reference, className, buttonText, id, title, }) { const capture = async () => { try { @@ -24,7 +24,7 @@ export default function MediaCaptureButton({ const base64image = canvas.toDataURL('image/png'); const a = document.createElement('a'); a.href = base64image; - a.setAttribute('download', ''); + a.setAttribute('download', `${title}.png`); a.click(); } catch (e) { // eslint-disable-next-line no-console @@ -50,6 +50,7 @@ MediaCaptureButton.propTypes = { className: PropTypes.string, buttonText: PropTypes.string, id: PropTypes.string.isRequired, + title: PropTypes.string.isRequired, }; MediaCaptureButton.defaultProps = { diff --git a/frontend/src/components/__tests__/MediaCaptureButton.js b/frontend/src/components/__tests__/MediaCaptureButton.js index aff2435fcf..48cddb0d2c 100644 --- a/frontend/src/components/__tests__/MediaCaptureButton.js +++ b/frontend/src/components/__tests__/MediaCaptureButton.js @@ -8,7 +8,7 @@ describe('MediaCaptureButton', () => { const RenderCaptureButton = () => { const widget = useRef(); return ( -
+
); }; it('renders', () => { diff --git a/frontend/src/widgets/DashboardOverview.js b/frontend/src/widgets/DashboardOverview.js index 5cefdde8e1..6d65075926 100644 --- a/frontend/src/widgets/DashboardOverview.js +++ b/frontend/src/widgets/DashboardOverview.js @@ -86,7 +86,7 @@ const DASHBOARD_FIELDS = { icon={faChartColumn} iconColor={colors.success} backgroundColor={colors.successLighter} - label={`across ${data.numReports} training reports`} + label={`across ${data.numReports} Training Reports`} data={`${data.numSessions} sessions`} /> ), diff --git a/frontend/src/widgets/TopicFrequencyGraph.js b/frontend/src/widgets/TopicFrequencyGraph.js index e67df87c9d..c1bbc92268 100644 --- a/frontend/src/widgets/TopicFrequencyGraph.js +++ b/frontend/src/widgets/TopicFrequencyGraph.js @@ -195,6 +195,7 @@ export function TopicFrequencyGraphWidget({ buttonText="Save screenshot" id="rd-save-screenshot-topic-frequency" className="margin-x-2" + title={title} /> ) : null} diff --git a/frontend/src/widgets/VBarGraph.js b/frontend/src/widgets/VBarGraph.js index 7b14d79e7f..33ae9bd12b 100644 --- a/frontend/src/widgets/VBarGraph.js +++ b/frontend/src/widgets/VBarGraph.js @@ -80,14 +80,19 @@ function VBarGraph({ automargin: true, autorange: true, tickangle: 0, - title: xAxisLabel, + title: { + text: xAxisLabel, + standoff: 40, + }, standoff: 20, }, yaxis: { tickformat: ',.0d', autorange: true, - title: yAxisLabel, - standoff: 50, + title: { + text: yAxisLabel, + standoff: 20, + }, }, hovermode: 'none', }; @@ -119,6 +124,7 @@ function VBarGraph({ reference={bars} buttonText="Save screenshot" id="rd-save-screenshot-vbars" + title={title} /> ) : null} diff --git a/frontend/src/widgets/__tests__/TrainingReportDashboardOverview.js b/frontend/src/widgets/__tests__/TrainingReportDashboardOverview.js index 49fd8121c8..fc4b053226 100644 --- a/frontend/src/widgets/__tests__/TrainingReportDashboardOverview.js +++ b/frontend/src/widgets/__tests__/TrainingReportDashboardOverview.js @@ -21,7 +21,7 @@ describe('TrainingReportDashboardOverview', () => { expect(screen.getAllByText('0%')).toHaveLength(1); expect(screen.getByText('Recipients have at least one active grant click to visually reveal this information')).toBeInTheDocument(); expect(screen.getByText('Grants served')).toBeInTheDocument(); - expect(screen.getByText('across 0 training reports')).toBeInTheDocument(); + expect(screen.getByText('across 0 Training Reports')).toBeInTheDocument(); expect(screen.getByText('Participants')).toBeInTheDocument(); expect(screen.getByText('Hours of TTA')).toBeInTheDocument(); }); @@ -47,7 +47,7 @@ describe('TrainingReportDashboardOverview', () => { expect(screen.getByText('Recipients have at least one active grant click to visually reveal this information')).toBeInTheDocument(); expect(screen.getByText('Grants served')).toBeInTheDocument(); - expect(screen.getByText('across 2 training reports')).toBeInTheDocument(); + expect(screen.getByText('across 2 Training Reports')).toBeInTheDocument(); expect(screen.getByText('Participants')).toBeInTheDocument(); expect(screen.getByText('Hours of TTA')).toBeInTheDocument(); });