Merge pull request #2418 from HHS/main

[PROD] Grant replacement. Auto-update GroupGrant on grant replacement.

README.md

@@ -119,7 +119,7 @@ If you are already using git hooks, add the .githooks/pre-commit contents to you
 It's important that our tests fully clean up after themselves if they interact with the database. This way, tests do not conflict when run on the CI and remain as deterministic as possible.The best way to do this is to run them locally in an isolated environment and confirm that they are sanitary.
 
 With that in mind, there a few "gotchas" to remember to help write sanitary tests.
-- ```Grant.destroy``` needs to run with ```individualHooks: true``` or the related GrantNumberLink model prevents delete.
+- ```Grant.destroy``` needs to run with ```individualHooks: true``` or the related GrantNumberLink model prevents delete. Additionally, the hooks on destroy also update the materialized view (GrantRelationshipToActive).
 - When you call ```Model.destroy``` you should be adding  ```individualHooks: true``` to the Sequelize options. Often this is required for proper cleanup. There may be times when this is undesirable; this should be indicated with a comment.
 - Be aware of paranoid models.  For those models: force: true gets around the soft delete. If they are already soft-deleted though, you need to remove the default scopes paranoid: true does it, as well as Model.unscoped()
 - There are excellent helpers for creating and destroying common Model mocks in ```testUtils.js```. Be aware that they take a scorched earth approach to cleanup. For example, when debugging a flaky test, it was discovered that ```destroyReport``` was removing a commonly used region.

docs/logical_data_model.puml

@@ -435,9 +435,27 @@ class GrantNumberLinks{
  deletedAt : timestamp with time zone
 }
 
+class GrantReplacementTypes{
+ * id : integer : <generated>
+ mapsTo : integer : REFERENCES "GrantReplacementTypes".id
+ * createdAt : timestamp with time zone : now()
+ * name : text
+ * updatedAt : timestamp with time zone : now()
+ deletedAt : timestamp with time zone
+}
+
+class GrantReplacements{
+ * id : integer : <generated>
+ grantReplacementTypeId : integer : REFERENCES "GrantReplacementTypes".id
+ * replacedGrantId : integer : REFERENCES "Grants".id
+ * replacingGrantId : integer : REFERENCES "Grants".id
+ * createdAt : timestamp with time zone : now()
+ * updatedAt : timestamp with time zone : now()
+ replacementDate : date
+}
+
 class Grants{
  * id : integer
- oldGrantId : integer : REFERENCES "Grants".id
  regionId : integer : REFERENCES "Regions".id
  * recipientId : integer : REFERENCES "Recipients".id
  * createdAt : timestamp with time zone : now()
@@ -1614,6 +1632,34 @@ class ZALGrantNumberLinks{
  session_sig : text
 }
 
+class ZALGrantReplacementTypes{
+ * id : bigint : <generated>
+ * data_id : bigint
+ * dml_as : bigint
+ * dml_by : bigint
+ * dml_timestamp : timestamp with time zone
+ * dml_txid : uuid
+ * dml_type : enum
+ descriptor_id : integer
+ new_row_data : jsonb
+ old_row_data : jsonb
+ session_sig : text
+}
+
+class ZALGrantReplacements{
+ * id : bigint : <generated>
+ * data_id : bigint
+ * dml_as : bigint
+ * dml_by : bigint
+ * dml_timestamp : timestamp with time zone
+ * dml_txid : uuid
+ * dml_type : enum
+ descriptor_id : integer
+ new_row_data : jsonb
+ old_row_data : jsonb
+ session_sig : text
+}
+
 class ZALGrants{
  * id : bigint : <generated>
  * data_id : bigint
@@ -2481,12 +2527,15 @@ Goals "1" --[#black,dashed,thickness=2]--{  "n" Objectives : objectives, goal
 Goals "1" --[#black,dashed,thickness=2]--{  "n" SimScoreGoalCaches : scoreOne, scoreTwo, goalOne, goalTwo
 GrantNumberLinks "1" --[#black,dashed,thickness=2]--{  "n" MonitoringClassSummaries : monitoringClassSummaries, grantNumberLink
 GrantNumberLinks "1" --[#black,dashed,thickness=2]--{  "n" MonitoringReviewGrantees : monitoringReviewGrantees, grantNumberLink
+GrantReplacementTypes "1" --[#black,dashed,thickness=2]--{  "n" GrantReplacementTypes : mapsFromReplacementType, mapsToReplacementType
+GrantReplacementTypes "1" --[#black,dashed,thickness=2]--{  "n" GrantReplacements : grantReplacements, grantReplacementType
 Grants "1" --[#black,dashed,thickness=2]--{  "n" ActivityRecipients : grant, activityRecipients
 Grants "1" --[#black,dashed,thickness=2]--{  "n" Goals : grant, goals
-Grants "1" --[#black,dashed,thickness=2]--{  "n" Grants : oldGrants, grant
+Grants "1" --[#black,dashed,thickness=2]--{  "n" GrantReplacements : replacedGrant, replacingGrant, replacedGrantReplacements, replacingGrantReplacements
 Grants "1" --[#black,dashed,thickness=2]--{  "n" GroupGrants : groupGrants, grant
 Grants "1" --[#black,dashed,thickness=2]--{  "n" ProgramPersonnel : programPersonnel, grant
 Grants "1" --[#black,dashed,thickness=2]--{  "n" Programs : programs, grant
+Grants "1" --[#black,dashed,thickness=2]--{  "n" undefined : grantRelationships, activeGrantRelationships
 Groups "1" --[#black,dashed,thickness=2]--{  "n" GroupCollaborators : group, groupCollaborators
 Groups "1" --[#black,dashed,thickness=2]--{  "n" GroupGrants : group, groupGrants
 ImportFiles "1" --[#black,dashed,thickness=2]--{  "n" ImportDataFiles : importFile, importDataFiles

src/goalServices/changeGoalStatus.test.js

@@ -55,7 +55,7 @@ describe('changeGoalStatus service', () => {
   afterAll(async () => {
     await db.Goal.destroy({ where: { id: goal.id }, force: true });
     await db.GrantNumberLink.destroy({ where: { grantId: grant.id }, force: true });
-    await db.Grant.destroy({ where: { id: grant.id } });
+    await db.Grant.destroy({ where: { id: grant.id }, individualHooks: true });
     await db.Recipient.destroy({ where: { id: recipient.id } });
     await db.UserRole.destroy({ where: { userId: user.id } });
     await db.Role.destroy({ where: { id: role.id } });

src/goalServices/getGoalIdsBySimiliarity.test.js

@@ -5,6 +5,8 @@ import {
   ActivityReportGoal,
   Grant,
   GrantNumberLink,
+  GrantReplacements,
+  GrantRelationshipToActive,
   Recipient,
   Goal,
   GoalTemplate,
@@ -80,9 +82,17 @@ describe('getGoalIdsBySimilarity', () => {
     replacementGrant = await createGrant({
       recipientId: recipient.id,
       status: 'Active',
-      oldGrantId: inactiveGrantWithReplacement.id,
     });
 
+    await GrantReplacements.create({
+      replacedGrantId: inactiveGrantWithReplacement.id,
+      replacingGrantId: replacementGrant.id,
+      replacementDate: new Date(),
+    });
+
+    // Refresh the materialized view.
+    await GrantRelationshipToActive.refresh();
+
     //  goals that will be ineligible for similarity
     // because they are on reports that have ineligible statuses
     const goalOnDraftReport = await createGoal({

src/goalServices/goals.js

@@ -12,6 +12,7 @@ import {
   GoalResource,
   GoalStatusChange,
   Grant,
+  GrantRelationshipToActive,
   Objective,
   ActivityReportObjective,
   sequelize,
@@ -69,6 +70,46 @@ const logContext = {
   namespace,
 };
 
+/**
+ * Maps grants to their active replacements.
+ *
+ * This function iterates through a list of grants and constructs a dictionary
+ * where each grant ID is associated with an array of active grant IDs. If the
+ * grant itself is active, it maps to its own ID. If the grant is not active,
+ * but has relationships with active grants, it maps to those active grants instead.
+ *
+ * @param {Array} grants - An array of grant objects. Each grant object should have
+ *   properties `id`, `status`, and an optional array of `grantRelationships`. Each
+ *   relationship should contain an `activeGrant` object with `id` and `status`.
+ * @returns {Object} A dictionary where the keys are grant IDs and the values are
+ *   arrays of active grant IDs related to each key grant.
+ */
+function mapGrantsWithReplacements(grants) {
+  const grantsWithReplacementsDictionary = {};
+
+  grants.forEach((grant) => {
+    if (grant.status === 'Active') {
+      if (Array.isArray(grantsWithReplacementsDictionary[grant.id])) {
+        grantsWithReplacementsDictionary[grant.id].push(grant.id);
+      } else {
+        grantsWithReplacementsDictionary[grant.id] = [grant.id];
+      }
+    } else {
+      grant.grantRelationships.forEach((relationship) => {
+        if (relationship.activeGrant && relationship.activeGrant.status === 'Active') {
+          if (Array.isArray(grantsWithReplacementsDictionary[grant.id])) {
+            grantsWithReplacementsDictionary[grant.id].push(relationship.activeGrantId);
+          } else {
+            grantsWithReplacementsDictionary[grant.id] = [relationship.activeGrantId];
+          }
+        }
+      });
+    }
+  });
+
+  return grantsWithReplacementsDictionary;
+}
+
 /**
  *
  * @param {number} id
@@ -621,20 +662,36 @@ export async function goalsForGrants(grantIds) {
   /**
    * get all the matching grants
    */
-  const grants = await Grant.findAll({
-    attributes: ['id', 'oldGrantId'],
+  const grants = await Grant.unscoped().findAll({
+    attributes: [
+      'id',
+      [sequelize.fn(
+        'ARRAY_AGG',
+        sequelize.fn(
+          'DISTINCT',
+          sequelize.col('grantRelationships.grantId'),
+        ),
+      ), 'oldGrantIds'],
+    ],
     where: {
       id: grantIds,
     },
+    include: [{
+      model: GrantRelationshipToActive,
+      as: 'grantRelationships',
+      required: false,
+      attributes: [],
+    }],
+    group: ['"Grant".id'],
   });
 
   /**
    * we need one big array that includes the old recipient id as well,
    * removing all the nulls along the way
    */
-  const ids = grants
+  const ids = Array.from(new Set(grants
     .reduce((previous, current) => [...previous, current.id, current.oldGrantId], [])
-    .filter((g) => g);
+    .filter((g) => g)));
 
   /*
   * finally, return all matching goals
@@ -660,7 +717,7 @@ export async function goalsForGrants(grantIds) {
         'ARRAY_AGG',
         sequelize.fn(
           'DISTINCT',
-          sequelize.col('grant.oldGrantId'),
+          sequelize.col('grant.grantRelationships.grantId'),
         ),
       ), 'oldGrantIds'],
       [sequelize.fn(
@@ -685,7 +742,7 @@ export async function goalsForGrants(grantIds) {
       'createdVia',
       [sequelize.fn('BOOL_OR', sequelize.literal(`"goalTemplate"."creationMethod" = '${CREATION_METHOD.CURATED}'`)), 'isCurated'],
     ],
-    group: ['"Goal"."name"', '"Goal"."status"', '"Goal"."endDate"', '"Goal"."onApprovedAR"', '"Goal"."source"', '"Goal"."createdVia"'],
+    group: ['"Goal"."name"', '"Goal"."status"', '"Goal"."endDate"', '"Goal"."onApprovedAR"', '"Goal"."source"', '"Goal"."createdVia"', '"Goal".id'],
     where: {
       name: {
         [Op.ne]: '', // exclude "blank" goals
@@ -700,6 +757,11 @@ export async function goalsForGrants(grantIds) {
         model: Grant.unscoped(),
         as: 'grant',
         attributes: [],
+        include: [{
+          model: GrantRelationshipToActive,
+          as: 'grantRelationships',
+          attributes: [],
+        }],
       },
       {
         model: GoalTemplate,
@@ -1645,23 +1707,31 @@ export async function getGoalIdsBySimilarity(recipientId, regionId, user = null)
   const uniqueGrantIds = uniq(goalGroups.map((group) => group.map((goal) => goal.grantId)).flat());
 
   const grants = await Grant.findAll({
+    attributes: ['id', 'status'],
     where: {
       [Op.or]: [
         { id: uniqueGrantIds },
-        { oldGrantId: uniqueGrantIds },
+        { '$grantRelationships.grantId$': uniqueGrantIds },
       ],
-      status: 'Active',
     },
-    attributes: ['id', 'oldGrantId', 'status'],
+    include: [
+      {
+        model: GrantRelationshipToActive,
+        as: 'grantRelationships',
+        attributes: ['activeGrantId', 'grantId'],
+        required: false,
+        include: [
+          {
+            model: Grant,
+            as: 'activeGrant',
+            attributes: ['id', 'status'],
+          },
+        ],
+      },
+    ],
   });
 
-  const grantLookup = {};
-  grants.forEach((grant) => {
-    grantLookup[grant.id] = grant.id;
-    if (grant.oldGrantId) {
-      grantLookup[grant.oldGrantId] = grant.id;
-    }
-  });
+  const grantsWithReplacementsDictionary = mapGrantsWithReplacements(grants);
 
   const filteredGoalGroups = goalGroups
     .filter((group) => {
@@ -1682,7 +1752,7 @@ export async function getGoalIdsBySimilarity(recipientId, regionId, user = null)
 
   const goalGroupsDeduplicated = filteredGoalGroups.map((group) => group
     .reduce((previous, current) => {
-      if (!grantLookup[current.grantId]) {
+      if (!grantsWithReplacementsDictionary[current.grantId]) {
         return previous;
       }
 
@@ -1719,7 +1789,7 @@ export async function getGoalIdsBySimilarity(recipientId, regionId, user = null)
           responsesForComparison: responsesForComparison(current),
           ids: [current.id],
           excludedIfNotAdmin,
-          grantId: grantLookup[current.grantId],
+          grantId: grantsWithReplacementsDictionary[current.grantId],
         },
       ];
     }, []));
@@ -1909,39 +1979,45 @@ export async function mergeGoals(
   const finalGoalValues = determineFinalGoalValues(selectedGoals, finalGoal);
 
   /**
-   * we will need to create a "new" final goal for each grant involved
-   * in this sordid business
-   */
+  * we will need to create a "new" final goal for each grant involved
+  * in this sordid business
+  */
 
   const uniqueGrantIds = uniq(selectedGoals.map((goal) => goal.grantId));
 
   const grantsWithReplacements = await Grant.findAll({
-    attributes: ['id', 'status', 'oldGrantId'],
+    attributes: ['id', 'status'],
     where: {
       [Op.or]: [
         { id: uniqueGrantIds },
-        { oldGrantId: uniqueGrantIds },
+        { '$grantRelationships.grantId$': uniqueGrantIds },
       ],
-      status: 'Active',
     },
+    include: [
+      {
+        model: GrantRelationshipToActive,
+        as: 'grantRelationships',
+        attributes: ['activeGrantId', 'grantId'],
+        required: false,
+        include: [
+          {
+            model: Grant,
+            as: 'activeGrant',
+            attributes: ['id', 'status'],
+          },
+        ],
+      },
+    ],
   });
 
   if (!grantsWithReplacements.length) {
     throw new Error('No active grants found to merge goals into');
   }
 
-  const grantsWithReplacementsDictionary = {};
-
-  grantsWithReplacements.forEach((grant) => {
-    if (grant.oldGrantId) {
-      grantsWithReplacementsDictionary[grant.oldGrantId] = grant.id;
-    }
-
-    grantsWithReplacementsDictionary[grant.id] = grant.id;
-  });
+  const grantsWithReplacementsDictionary = mapGrantsWithReplacements(grantsWithReplacements);
 
   // unique list of grant IDs
-  const grantIds = uniq(grantsWithReplacements.map((grant) => grant.id));
+  const grantIds = uniq(Object.values(grantsWithReplacementsDictionary).flat());
 
   const goalsToBulkCreate = grantIds.map((grantId) => ({
     ...finalGoalValues,
@@ -2008,7 +2084,8 @@ export async function mergeGoals(
     // will use the most up-to-date grant ID as well
     mergeObjectivesFromGoal(g, grantToGoalDictionary[
       grantsWithReplacementsDictionary[g.grantId]
-    ]))));
+    ])
+  )));
 
   const updatesToRelatedModels = [];