diff --git a/.circleci/config.yml b/.circleci/config.yml index 093dc0b56f..115b486bfa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,6 +70,15 @@ commands: description: "Name of CircleCI project environment variable that holds the New Relic License key, a required application variable" type: env_var_name + hses_data_file_url: + description: "Url to download HSES grants and grantee data from" + type: env_var_name + hses_data_username: + description: "Username used to access the HSES grants and grantee data" + type: env_var_name + hses_data_password: + description: "Password used to access the HSES grants and grantee data" + type: env_var_name steps: - run: name: Login with service account @@ -86,7 +95,10 @@ commands: --var AUTH_CLIENT_ID=${<< parameters.auth_client_id >>} \ --var AUTH_CLIENT_SECRET=${<< parameters.auth_client_secret >>} \ --var NEW_RELIC_LICENSE_KEY=${<< parameters.new_relic_license >>} \ - --var SESSION_SECRET=${<< parameters.session_secret >>} + --var SESSION_SECRET=${<< parameters.session_secret >>} \ + --var HSES_DATA_FILE_URL=${<< parameters.hses_data_file_url >>} \ + --var HSES_DATA_USERNAME=${<< parameters.hses_data_username >>} \ + --var HSES_DATA_PASSWORD=${<< parameters.hses_data_password >>} parameters: cg_org: description: "Cloud Foundry cloud.gov organization name" @@ -121,7 +133,7 @@ parameters: default: "main" type: string sandbox_git_branch: # change to feature branch to test deployment - default: "js-fix-user-autoinc" + default: "kw-pull-hs-data" type: string jobs: build_and_lint: @@ -168,6 +180,9 @@ jobs: - run: name: Run migrations ci command: yarn db:migrate:ci + - run: + name: Run seeders + command: yarn db:seed:ci - run: name: Test backend command: yarn test:ci @@ -302,6 +317,9 @@ jobs: deploy_config_file: deployment_config/sandbox_vars.yml new_relic_license: NEW_RELIC_LICENSE_KEY session_secret: SANDBOX_SESSION_SECRET + hses_data_file_url: HSES_DATA_FILE_URL + hses_data_username: HSES_DATA_USERNAME + hses_data_password: HSES_DATA_PASSWORD - run: name: Return database to neutral, then migrate and seed command: | @@ -323,6 +341,9 @@ jobs: deploy_config_file: deployment_config/dev_vars.yml new_relic_license: NEW_RELIC_LICENSE_KEY session_secret: DEV_SESSION_SECRET + hses_data_file_url: HSES_DATA_FILE_URL + hses_data_username: HSES_DATA_USERNAME + hses_data_password: HSES_DATA_PASSWORD - run: name: Undo database seeding, then migrate and seed command: | @@ -343,6 +364,9 @@ jobs: deploy_config_file: deployment_config/staging_vars.yml new_relic_license: NEW_RELIC_LICENSE_KEY session_secret: STAGING_SESSION_SECRET + hses_data_file_url: HSES_DATA_FILE_URL + hses_data_username: HSES_DATA_USERNAME + hses_data_password: HSES_DATA_PASSWORD - run: name: Run database migrations command: | @@ -363,6 +387,9 @@ jobs: deploy_config_file: deployment_config/prod_vars.yml new_relic_license: PROD_NEW_RELIC_LICENSE_KEY session_secret: PROD_SESSION_SECRET + hses_data_file_url: PROD_HSES_DATA_FILE_URL + hses_data_username: PROD_HSES_DATA_USERNAME + hses_data_password: PROD_HSES_DATA_PASSWORD - run: name: Run database migrations command: | diff --git a/.env.example b/.env.example index 3968573c04..2df2887f29 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,9 @@ POSTGRES_USERNAME=postgres POSTGRES_PASSWORD=something_secret POSTGRES_DB=ttasmarthub POSTGRES_HOST=localhost +# Add Oauth client id for local development from "Development Credentials" document here. AUTH_CLIENT_ID=clientId +# Add Oauth client secret for local development from "Development Credentials" document here. AUTH_CLIENT_SECRET=clientSecret SESSION_SECRET=secret TTA_SMART_HUB_URI=http://localhost:3000 @@ -19,11 +21,14 @@ AUTH_BASE=https://uat.hsesinfo.org REDIRECT_URI_HOST=http://localhost:8080 # CURRENT_USER_ID controls the logged in user when BYPASS_AUTH is set to true. # This only works in non-production environments -CURRENT_USER_ID=1; +CURRENT_USER_ID=1 # NEW_RELIC_LICENSE_KEY can be omitted in local development NEW_RELIC_LICENSE_KEY=secret_key # Set to false to require user to go through auth flow, never true in production envs BYPASS_AUTH=true +HSES_DATA_FILE_URL=url +HSES_DATA_USERNAME=username +HSES_DATA_PASSWORD=password # In production, Sequelize instance is created with a postgres URI. # This URI is automatically dropped into the cloud.gov environment as the env variable DATABASE_URL DATABASE_URL=secret diff --git a/README.md b/README.md index e5b22304b7..b7cffddcc4 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,27 @@ For the latest on our product mission, goals, initiatives, and KPIs, see the [Pr ## Getting Started ### Set up - -Make sure Docker is installed. To check run `docker ps` - -Run `yarn docker:deps`. This builds the frontend and backend docker containers and install dependencies. You only need to run this step the first time you fire up the app and when dependencies are added/updated/removed. Running `yarn docker:start` starts the backend and frontend, browse to `http://localhost:3000` to hit the frontend and `http://localhost:3000/api` to hit the backend. Copying `.env.example` to `.env`, substituting in your user id and group id will cause any files created in docker containers to be owned by your user on your host. - -You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`. +#### Docker + +1. Make sure Docker is installed. To check run `docker ps`. +2. Make sure you have Node 12.20.0 installed. +3. Run `yarn docker:deps`. This builds the frontend and backend docker containers and install dependencies. You only need to run this step the first time you fire up the app and when dependencies are added/updated/removed. +4. Copy `.env.example` to `.env`. +6. Change the `AUTH_CLIENT_ID` and `AUTH_CLIENT_SECRET` variables to to values found in the "Values for local development" section of the "Development Credentials" document. If you don't have access to this document, please ask in the hs-vendors-ohs-tta channel of the gsa-tts slack channel. +7. Optionally, set `CURRENT_USER` to your current user's uid:gid. This will cause files created by docker compose to be owned by your user instead of root. +8. Run `yarn docker:db:migrate` to run DB migrations +9. Run `yarn docker:db:seed` to seed the database with test data. +10. Run `yarn docker:start` to start the application. The frontend will be available on `localhost:3000` and the backend will run on `localhost:8080`. +11. Run `yarn docker:stop` to stop the servers and remove the docker containers. The frontend [proxies requests](https://create-react-app.dev/docs/proxying-api-requests-in-development/) to paths it doesn't recognize to the backend. Api documentation uses [Redoc](https://github.com/Redocly/redoc) to serve documentation files. These files can be found in the `docs/openapi` folder. Api documentation should be split into separate files when appropriate to prevent huge hard to grasp yaml files. +#### Local build + +You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`. + ### Running Tests Run `yarn docker:deps` to install dependencies. Run `yarn docker:db:migrate` and `yarn docker:test` to run all tests for the frontend and backend. @@ -215,6 +225,18 @@ Our project includes four deployed Postgres databases, one to interact with each You can run psql commands directly against a deployed database by following these directions. +1. Install Cloud Foundry CLI tool + + - On MacOS: `brew install cloudfoundry/tap/cf-cli` + - On other platforms: [Download and install cf][cf-install] + +1. Login to cloud.gov account + + ```bash + cf login -a api.fr.cloud.gov --sso + # follow temporary authorization code prompts + ``` + 1. Install the cloud foundry plugin [cf-service-connect][cf-service-connect] ```bash @@ -247,6 +269,7 @@ You can run psql commands directly against a deployed database by following thes [circleci-envvar]: https://app.circleci.com/settings/project/github/adhocteam/Head-Start-TTADP/environment-variables?return-to=https%3A%2F%2Fcircleci.com%2Fdashboard [cloudgov]: https://dashboard.fr.cloud.gov/home [cloudgov-deployer]: https://cloud.gov/docs/services/cloud-gov-service-account/ +[cf-install]: https://docs.cloudfoundry.org/cf-cli/install-go-cli.html [cf-service-connect]: https://github.com/cloud-gov/cf-service-connect [hhs-main]: https://github.com/HHS/Head-Start-TTADP/tree/main [hhs-prod]: https://github.com/HHS/Head-Start-TTADP/tree/production diff --git a/hses.zip b/hses.zip new file mode 100644 index 0000000000..21acb9c402 Binary files /dev/null and b/hses.zip differ diff --git a/manifest.yml b/manifest.yml index e7e6d6b49f..4db4791985 100644 --- a/manifest.yml +++ b/manifest.yml @@ -16,6 +16,9 @@ applications: REDIRECT_URI_HOST: ((REDIRECT_URI_HOST)) SESSION_SECRET: ((SESSION_SECRET)) TTA_SMART_HUB_URI: ((TTA_SMART_HUB_URI)) + HSES_DATA_FILE_URL: ((HSES_DATA_FILE_URL)) + HSES_DATA_USERNAME: ((HSES_DATA_USERNAME)) + HSES_DATA_PASSWORD: ((HSES_DATA_PASSWORD)) services: - ((rds_instance)) - ((s3_doc_upload_bucket)) diff --git a/package.json b/package.json index 7cd280b21b..560d2b6447 100644 --- a/package.json +++ b/package.json @@ -132,9 +132,11 @@ "dependencies": { "@babel/runtime": "^7.12.1", "@cucumber/cucumber": "^7.0.0-rc.0", + "adm-zip": "^0.5.1", "axios": "^0.21.1", "chromedriver": "^87.0.0", "client-oauth2": "^4.3.3", + "cron": "^1.8.2", "csv-parse": "^4.14.1", "cucumber-html-reporter": "^5.2.0", "dotenv": "^8.2.0", @@ -146,6 +148,7 @@ "http-codes": "^1.0.0", "lodash": "^4.17.20", "memorystore": "^1.6.2", + "mz": "^2.7.0", "newrelic": "^7.0.1", "pg": "^8.3.3", "puppeteer": "^5.3.1", @@ -154,6 +157,7 @@ "sequelize-cli": "^6.2.0", "url-join": "^4.0.1", "winston": "^3.3.3", + "xml2json": "^0.12.0", "yargs": "^16.1.1" } } diff --git a/src/app.js b/src/app.js index 7f3603cdbe..6d4738f588 100644 --- a/src/app.js +++ b/src/app.js @@ -7,7 +7,9 @@ import memorystore from 'memorystore'; import path from 'path'; import join from 'url-join'; import { INTERNAL_SERVER_ERROR } from 'http-codes'; +import { CronJob } from 'cron'; import { hsesAuth } from './middleware/authMiddleware'; +import updateGrantsGrantees from './lib/updateGrantsGrantees'; import findOrCreateUser from './services/accessValidation'; @@ -77,4 +79,23 @@ if (process.env.NODE_ENV === 'production') { }); } +// Set timing parameters. +// Run at midnight +const schedule = '0 0 * * *'; +const timezone = 'America/New_York'; + +const runJob = () => { + try { + updateGrantsGrantees(); + } catch (error) { + logger.error(`Error processing HSES file: ${error}`); + } +}; + +// Run only on one instance +if (process.env.CF_INSTANCE_INDEX === '0') { + const job = new CronJob(schedule, () => runJob(), null, true, timezone); + job.start(); +} + module.exports = app; diff --git a/src/lib/updateGrantsGrantees.js b/src/lib/updateGrantsGrantees.js new file mode 100644 index 0000000000..41eaa1aa39 --- /dev/null +++ b/src/lib/updateGrantsGrantees.js @@ -0,0 +1,124 @@ +import AdmZip from 'adm-zip'; +import { toJson } from 'xml2json'; +import {} from 'dotenv/config'; +import axios from 'axios'; +import { + Grantee, Grant, +} from '../models'; +import logger from '../logger'; + +const fs = require('mz/fs'); +/** + * Reads HSES data files that were previously extracted to the "temp" directory. + * The files received from HSES are: + * + * agency.xml - grantee and grantee that are delegates + * grant_agency.xml - junction between grants and agencies + * grant_award.xml - grants + * grant_award_replacement.xml + * grant_program.xml + * + * The grantee data is them filtered to exclude delegates + * + */ +export async function processFiles() { + let grantGrantees; + let grants; + const granteesForDb = []; + const grantsForDb = []; + + try { + const grantAgencyData = await fs.readFile('./temp/grant_agency.xml'); + const json = toJson(grantAgencyData); + const grantAgency = JSON.parse(json); + // we are only interested in non-delegates + grantGrantees = grantAgency.grant_agencies.grant_agency.filter( + (g) => g.grant_agency_number === '0', + ); + + // process grantees aka agencies that are non-delegates + const agencyData = await fs.readFile('./temp/agency.xml'); + const agency = JSON.parse(toJson(agencyData)); + + // filter out delegates by matching to the non-delegates + // eslint-disable-next-line max-len + const granteesNonDelegates = agency.agencies.agency.filter((a) => grantGrantees.some((gg) => gg.agency_id === a.agency_id)); + + const hubGranteeIds = await Grantee.findAll({ attributes: ['id'] }).map((hgi) => hgi.id); + + // process grants + const grantData = await fs.readFile('./temp/grant_award.xml'); + const grant = JSON.parse(toJson(grantData)); + + // Check if the grantee id already exists in the smarthub db OR if it belongs to + // at least one active grant. grant_award data structure includes agency_id + // eslint-disable-next-line max-len + const grantees = granteesNonDelegates.filter((gnd) => hubGranteeIds.some((id) => id.toString() === gnd.agency_id) + || grant.grant_awards.grant_award.some((ga) => ga.agency_id === gnd.agency_id && ga.grant_status === 'Active')); + + grantees.forEach((g) => granteesForDb.push({ + id: parseInt(g.agency_id, 10), + name: g.agency_name, + })); + + await Grantee.bulkCreate(granteesForDb, + { + updateOnDuplicate: ['name', 'updatedAt'], + }); + + const hubGrantIds = await Grant.findAll({ attributes: ['id'] }).map((hgi) => hgi.id); + + grants = grant.grant_awards.grant_award.filter((ga) => hubGrantIds.some((id) => id.toString() === ga.grant_award_id) || ga.grant_status === 'Active'); + + grants.forEach((g) => grantsForDb.push({ + id: parseInt(g.grant_award_id, 10), + number: g.grant_number, + regionId: parseInt(g.region_id, 10), + granteeId: parseInt(g.agency_id, 10), + status: g.grant_status, + startDate: g.grant_start_date, + endDate: g.grant_end_date, + })); + + await Grant.bulkCreate(grantsForDb, + { + updateOnDuplicate: ['number', 'regionId', 'granteeId', 'status', 'startDate', 'endDate', 'updatedAt'], + }); + } catch (error) { + logger.error(`Error reading or updating database on HSES data import: ${error.message}`); + throw error; + } +} + +// reading archives +const zip = new AdmZip('./hses.zip'); + +/** + * Downloads the HSES grantee/grant zip, extracts to the "temp" directory + * and calls processFiles to parse xml data and populate the Smart Hub db + * + * Note - file download needs to happen in deployed environments + */ +export default async function updateGrantsGrantees() { + try { + if (process.env.NODE_ENV === 'production') { + const response = await axios(process.env.HSES_DATA_FILE_URL, { + method: 'get', + url: process.env.HSES_DATA_FILE_URL, + responseType: 'stream', + auth: { + username: process.env.HSES_DATA_USERNAME, + password: process.env.HSES_DATA_PASSWORD, + }, + }); + + await response.data.pipe(fs.createWriteStream('hses.zip')); + } + // extract to target path. Pass true to overwrite + zip.extractAllTo('./temp', true); + + await processFiles(); + } catch (error) { + logger.error(error); + } +} diff --git a/src/migrations/20201029214432-add-title-to-user.js b/src/migrations/20201029214432-add-title-to-user.js index 775067df9a..f912d54397 100644 --- a/src/migrations/20201029214432-add-title-to-user.js +++ b/src/migrations/20201029214432-add-title-to-user.js @@ -23,6 +23,7 @@ module.exports = { 'homeRegionId', { type: Sequelize.INTEGER, + allowNull: true, references: { model: 'Regions', key: 'id', diff --git a/src/migrations/20201130144748-create-grantee.js b/src/migrations/20201130144748-create-grantee.js index 42767226aa..0f818b89d8 100644 --- a/src/migrations/20201130144748-create-grantee.js +++ b/src/migrations/20201130144748-create-grantee.js @@ -3,7 +3,7 @@ module.exports = { await queryInterface.createTable('Grantees', { id: { allowNull: false, - autoIncrement: true, + autoIncrement: false, primaryKey: true, type: Sequelize.INTEGER, }, @@ -13,10 +13,12 @@ module.exports = { createdAt: { allowNull: false, type: Sequelize.DATE, + defaultValue: Sequelize.fn('NOW'), }, updatedAt: { allowNull: false, type: Sequelize.DATE, + defaultValue: Sequelize.fn('NOW'), }, }); }, diff --git a/src/migrations/20201205200637-create-grant.js b/src/migrations/20201205200637-create-grant.js index bad89f8e27..1845c43a6b 100644 --- a/src/migrations/20201205200637-create-grant.js +++ b/src/migrations/20201205200637-create-grant.js @@ -3,7 +3,7 @@ module.exports = { await queryInterface.createTable('Grants', { id: { allowNull: false, - autoIncrement: true, + autoIncrement: false, primaryKey: true, type: Sequelize.INTEGER, }, @@ -44,10 +44,12 @@ module.exports = { createdAt: { allowNull: false, type: Sequelize.DATE, + defaultValue: Sequelize.fn('NOW'), }, updatedAt: { allowNull: false, type: Sequelize.DATE, + defaultValue: Sequelize.fn('NOW'), }, }); }, diff --git a/src/migrations/20210105154127-users-home-region-allow-null.js b/src/migrations/20210105154127-users-home-region-allow-null.js deleted file mode 100644 index d8f688e1a4..0000000000 --- a/src/migrations/20210105154127-users-home-region-allow-null.js +++ /dev/null @@ -1,15 +0,0 @@ -module.exports = { - up: async (queryInterface, Sequelize) => { - await queryInterface.changeColumn('Users', 'homeRegionId', { - type: Sequelize.INTEGER, - allowNull: true, - }); - }, - - down: async (queryInterface, Sequelize) => { - await queryInterface.changeColumn('Users', 'homeRegionId', { - type: Sequelize.INTEGER, - allowNull: false, - }); - }, -}; diff --git a/src/seeders/20201210172017-grantees.js b/src/seeders/20201210172017-grantees.js new file mode 100644 index 0000000000..725c3e9545 --- /dev/null +++ b/src/seeders/20201210172017-grantees.js @@ -0,0 +1,44 @@ +const grantees = [ + { + id: 1, + name: 'Grantee Name', + }, + { + id: 2, + name: 'Stroman, Cronin and Boehm', + }, + { + id: 3, + name: 'Jakubowski-Keebler', + }, + { + id: 4, + name: 'Johnston-Romaguera', + }, + { + id: 5, + name: 'Agency 1, Inc.', + }, + { + id: 6, + name: 'Agency 2, Inc.', + }, + { + id: 7, + name: 'Agency 3, Inc.', + }, + { + id: 8, + name: 'Agency 4, Inc.', + }, +]; + +module.exports = { + up: async (queryInterface) => { + await queryInterface.bulkInsert('Grantees', grantees, {}); + }, + + down: async (queryInterface) => { + await queryInterface.bulkDelete('Grantees', null, {}); + }, +}; diff --git a/src/seeders/20201211172017-grants.js b/src/seeders/20201211172017-grants.js new file mode 100644 index 0000000000..208d855436 --- /dev/null +++ b/src/seeders/20201211172017-grants.js @@ -0,0 +1,66 @@ +const grants = [ + { + id: 1, + number: '14CH1234', + regionId: 14, + granteeId: 1, + }, + { + id: 2, + number: '14CH10000', + regionId: 14, + granteeId: 2, + }, + { + id: 3, + number: '14CH00001', + regionId: 14, + granteeId: 3, + }, + { + id: 4, + number: '14CH00002', + regionId: 14, + granteeId: 4, + }, + { + id: 5, + number: '14CH00003', + regionId: 14, + granteeId: 4, + }, + { + id: 6, + number: '09CH011111', + regionId: 9, + granteeId: 5, + }, + { + id: 7, + number: '09CH022222', + regionId: 9, + granteeId: 6, + }, + { + id: 8, + number: '09CH033333', + regionId: 9, + granteeId: 7, + }, + { + id: 9, + number: '09HP044444', + regionId: 9, + granteeId: 8, + }, +]; + +module.exports = { + up: async (queryInterface) => { + await queryInterface.bulkInsert('Grants', grants, {}); + }, + + down: async (queryInterface) => { + await queryInterface.bulkDelete('Grants', null, {}); + }, +}; diff --git a/temp/agency.xml b/temp/agency.xml new file mode 100644 index 0000000000..54b0ce15b6 --- /dev/null +++ b/temp/agency.xml @@ -0,0 +1,22 @@ + + + 1335 + Agency 1, Inc. + Community Action Agency (CAA) + + + 4365 + Community College + School System + + + 119 + Multi ID Agency + Tribal Government or Consortium + + + 7709 + Multi ID Agency + + + \ No newline at end of file diff --git a/temp/grant_agency.xml b/temp/grant_agency.xml new file mode 100644 index 0000000000..61fff8eb6b --- /dev/null +++ b/temp/grant_agency.xml @@ -0,0 +1,62 @@ + + + 8647 + 1335 + 7842 + 0 + + + 8940 + 1335 + 8110 + 0 + + + 2853 + 1335 + 2591 + 0 + + + 13146 + 1335 + 11835 + 0 + + + 11509 + 1335 + 10448 + 0 + + + 6181 + 1335 + 6223 + 0 + + + 11636 + 1335 + 10567 + 0 + + + 5224 + 119 + 5151 + 0 + + + 9393 + 119 + 8564 + 0 + + + 12924 + 7709 + 11628 + 0 + + \ No newline at end of file diff --git a/temp/grant_award.xml b/temp/grant_award.xml new file mode 100644 index 0000000000..ba6f81f77e --- /dev/null +++ b/temp/grant_award.xml @@ -0,0 +1,112 @@ + + + 7842 + 1335 + 2 + 02CH01105 + 2014-03-01 + 2019-02-28 + CH + Agency 1, Inc. + Inactive + + + 8110 + 1335 + 2 + 02CH01106 + 2014-07-01 + 2020-04-30 + CH + Agency 1, Inc. + Inactive + + + 2591 + 1335 + 2 + 02CH01107 + 2013-07-01 + 2018-11-30 + CH + Agency 1, Inc. + Inactive + + + 11835 + 1335 + 2 + 02CH01108 + 2020-05-01 + 2025-04-30 + CH + Agency 1, Inc. + Active + + + 10448 + 1335 + 2 + 02CH01109 + 2018-12-01 + 2023-11-30 + CH + Agency 1, Inc. + Active + + + 6223 + 1335 + 2 + 02CH01110 + 1992-03-01 + 2025-06-30 + CH + BERGEN COUNTY COMMUNITY ACTION PROGRAM, INC + Inactive + + + 10567 + 1335 + 2 + 02CH01111 + 2019-03-01 + 2024-02-29 + CH + Agency 1, Inc. + Active + + + 5151 + 119 + 11 + 90CI4444 + 1994-08-01 + 2025-06-30 + CI + Multi ID Agency + Inactive + + + 8564 + 119 + 11 + 90CI5555 + 2015-01-01 + 2019-12-31 + CI + Multi ID Agency + Inactive + + + 11628 + 7709 + 11 + 90CI090022 + 2020-01-01 + 2024-12-31 + CI + Multi ID Agency + Active + + \ No newline at end of file diff --git a/tools/importPlanGoals.js b/tools/importPlanGoals.js index f77cb99a55..3621d3585d 100644 --- a/tools/importPlanGoals.js +++ b/tools/importPlanGoals.js @@ -5,6 +5,7 @@ import parse from 'csv-parse/lib/sync'; import { Role, Topic, RoleTopic, Goal, TopicGoal, Grantee, Grant, GrantGoal, } from '../src/models'; +import { exit } from 'process'; const hubRoles = [ { name: 'RPM', fullName: 'Regional Program Manager' }, @@ -64,16 +65,13 @@ export default async function importGoals(file, region) { const regionId = region || 14; // default to region 14 try { const cleanRoleTopics = []; - const cleanGrantees = []; const cleanGrantGoals = []; const cleanTopicGoals = []; - const cleanGrants = []; const currentGoals = []; await prePopulateRoles(); for await (const el of grantees) { - let currentGrantee; let currentGranteeId; let grants; let currentGrants = []; @@ -82,10 +80,6 @@ export default async function importGoals(file, region) { for await (const key of Object.keys(el)) { if (key && (key.trim().startsWith('Grantee (distinct') || key.trim().startsWith('Grantee Name'))) { - currentGrantee = el[key] ? el[key].split('|')[0].trim() : 'Unknown Grantee'; - const [dbGrantee] = await Grantee.findOrCreate({ where: { name: currentGrantee } }); - currentGranteeId = dbGrantee.id; - cleanGrantees.push({ id: currentGranteeId, name: currentGrantee }); grants = el[key] ? el[key].split('|')[1].trim() : 'Unknown Grant'; currentGrants = grants.split(','); } else if (key && key.startsWith('Goal')) { @@ -163,15 +157,15 @@ export default async function importGoals(file, region) { tp.goalId = goalId; } }); - currentGranteeId = cleanGrantees.find((g) => g.name === currentGrantee).id; for await (const grant of currentGrants) { - const fullGrant = { number: grant.trim(), granteeId: currentGranteeId }; - if (!cleanGrants.some((e) => e.granteeId === fullGrant.granteeId - && e.number === fullGrant.number)) { - cleanGrants.push(fullGrant); + const fullGrant = { number: grant.trim(), regionId }; + const dbGrant = await Grant.findOne({ where: { ...fullGrant }, attributes: ['id', 'granteeId'] }); + if (!dbGrant) { + console.log(`Couldn't find grant: ${fullGrant.number}. Exiting...`); + process.exit(1); } - const [dbGrant] = await Grant.findOrCreate({ where: { ...fullGrant, regionId } }); grantId = dbGrant.id; + currentGranteeId = dbGrant.granteeId; const plan = { granteeId: currentGranteeId, grantId, goalId }; if (!cleanGrantGoals.some((e) => e.granteeId === currentGranteeId && e.grantId === grantId diff --git a/tools/importPlanGoals.test.js b/tools/importPlanGoals.test.js index bdb6ad4042..500b9a9510 100644 --- a/tools/importPlanGoals.test.js +++ b/tools/importPlanGoals.test.js @@ -1,12 +1,80 @@ +import { Op } from 'sequelize'; import importGoals from './importPlanGoals'; +import { processFiles } from '../src/lib/updateGrantsGrantees'; import db, { - Role, Topic, RoleTopic, Goal, Grantee, Grant, sequelize, + Role, Topic, RoleTopic, Goal, Grantee, Grant, } from '../src/models'; describe('Import TTA plan goals', () => { afterAll(() => { db.sequelize.close(); }); + describe('Update grants and grantees', () => { + beforeAll(async () => { + await Grant.destroy({ where: { id: { [Op.gt]: 20 } } }); + await Grantee.destroy({ where: { id: { [Op.gt]: 20 } } }); + }); + afterEach(async () => { + await Grant.destroy({ where: { id: { [Op.gt]: 20 } } }); + await Grantee.destroy({ where: { id: { [Op.gt]: 20 } } }); + }); + it('should import or update grantees', async () => { + const granteesBefore = await Grantee.findAll({ where: { id: { [Op.gt]: 20 } } }); + expect(granteesBefore.length).toBe(0); + await processFiles(); + + const grantee = await Grantee.findOne({ where: { id: 1335 } }); + expect(grantee).toBeDefined(); + expect(grantee.name).toBe('Agency 1, Inc.'); + }); + + it('should import or update grants', async () => { + const grantsBefore = await Grant.findAll({ where: { id: { [Op.gt]: 20 } } }); + + expect(grantsBefore.length).toBe(0); + await processFiles(); + + const grants = await Grant.findAll({ where: { granteeId: 1335 } }); + expect(grants).toBeDefined(); + expect(grants.length).toBe(3); + const containsNumber = grants.some((g) => g.number === '02CH01111'); + expect(containsNumber).toBeTruthy(); + }); + + it('should exclude grantees with only inactive grants', async () => { + await processFiles(); + let grantee = await Grantee.findOne({ where: { id: 119 } }); + expect(grantee).toBeNull(); + // Same grantee, but with a different id and having an active grant + grantee = await Grantee.findOne({ where: { id: 7709 } }); + expect(grantee.name).toBe('Multi ID Agency'); + }); + + it('should update an existing grantee if it exists in smarthub', async () => { + const [dbGrantee] = await Grantee.findOrCreate({ where: { id: 119, name: 'Multi ID Agency' } }); + await processFiles(); + const grantee = await Grantee.findOne({ where: { id: 119 } }); + expect(grantee).not.toBeNull(); + // Same grantee, but with a different id and having an active grant + expect(grantee.updatedAt).not.toEqual(dbGrantee.updatedAt); + expect(grantee.name).toBe('Multi ID Agency'); + }); + + it('should update an existing grant if it exists in smarthub', async () => { + await processFiles(); + let grant = await Grant.findOne({ where: { id: 5151 } }); + expect(grant).toBeNull(); + + await Grantee.findOrCreate({ where: { id: 119, name: 'Multi ID Agency' } }); + const [dbGrant] = await Grant.findOrCreate({ where: { id: 5151, number: '90CI4444', granteeId: 119 } }); + await processFiles(); + grant = await Grant.findOne({ where: { id: 5151 } }); + expect(grant).not.toBeNull(); + expect(grant.updatedAt).not.toEqual(dbGrant.updatedAt); + expect(grant.number).toBe('90CI4444'); + }); + }); + it('should import Roles table', async () => { await Role.destroy({ where: {} }); const rolesBefore = await Role.findAll(); @@ -165,17 +233,12 @@ describe('Import TTA plan goals', () => { ); }); - it('should import Grantees table', async () => { - await Grant.destroy({ where: [] }); - await Grantee.destroy({ where: {} }); - const granteesBefore = await Grantee.findAll(); - - expect(granteesBefore.length).toBe(0); - - await importGoals('GranteeTTAPlanTest.csv'); + it('should have Grantees Goals connection', async () => { const grantees = await Grantee.findAll(); expect(grantees).toBeDefined(); - expect(grantees.length).toBe(4); + expect(grantees.length).toBe(8); + + await importGoals('GranteeTTAPlanTest.csv'); // test eager loading const grantee = await Grantee.findOne({ @@ -196,20 +259,6 @@ describe('Import TTA plan goals', () => { expect(grantee.goals[1].name).toEqual('Enhance reflective practice.'); }); - it('should import Grants table', async () => { - await Grant.destroy({ where: {} }); - const grantsBefore = await Grant.findAll(); - - expect(grantsBefore.length).toBe(0); - await importGoals('GranteeTTAPlanTest.csv'); - - const grants = await Grant.findAll(); - expect(grants).toBeDefined(); - expect(grants.length).toBe(5); - expect(grants[1].number).toBe('14CH10000'); - expect(grants[1].regionId).toBe(14); - }); - it('should import RoleTopics table', async () => { await Topic.destroy({ where: {} }); @@ -293,22 +342,6 @@ describe('Import TTA plan goals', () => { }, }], }], - // { - // model: Grantee, - // as: 'grantees', - // attributes: ['id', 'name'], - // through: { - // attributes: [], - // }, - // }, - // { - // model: Grant, - // as: 'grants', - // attributes: ['id', 'number', 'regionId'], - // through: { - // attributes: [], - // }, - // }], }); expect(goalWithTopic.topics[0].roles[0].fullName).toBe('Grantee Specialist'); }); diff --git a/yarn.lock b/yarn.lock index be53ea81d9..bd3eb15fe8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1819,6 +1819,11 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== +adm-zip@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.1.tgz#32e51c6fe88370f4389b424567b78a345e79ebc6" + integrity sha512-a5ABmIFUJ9OxHV5zrXM9Q41JzpRIflFtdgpL4UQM9DsTHHxQzPRaeyAdnMW7kxL0NRWm/NHafJdj6pO+ty7L2g== + agent-base@5: version "5.1.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c" @@ -3189,6 +3194,13 @@ create-require@^1.1.0: resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== +cron@^1.8.2: + version "1.8.2" + resolved "https://registry.yarnpkg.com/cron/-/cron-1.8.2.tgz#4ac5e3c55ba8c163d84f3407bde94632da8370ce" + integrity sha512-Gk2c4y6xKEO8FSAUTklqtfSr7oTq0CiPQeLBG5Fl0qoXpZyMcj1SG59YL+hqq04bu6/IuEA7lMkYDAplQNKkyg== + dependencies: + moment-timezone "^0.5.x" + cross-env@^7.0.2: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" @@ -4935,6 +4947,21 @@ hmac-drbg@^1.0.0: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" +hoek@5.x.x: + version "5.0.4" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-5.0.4.tgz#0f7fa270a1cafeb364a4b2ddfaa33f864e4157da" + integrity sha512-Alr4ZQgoMlnere5FZJsIyfIjORBqZll5POhDsF4q64dPuJR6rNxXdDxtHSQq8OXRurhmx+PWYEE8bXRROY8h0w== + +hoek@6.x.x: + version "6.1.3" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-6.1.3.tgz#73b7d33952e01fe27a38b0457294b79dd8da242c" + integrity sha512-YXXAAhmF9zpQbC7LEcREFtXfGq5K1fmd+4PHkBq8NUqmzW3G+Dq10bI/i0KucLRwss3YYFQ0fSfoxBZYiGUqtQ== + +hoek@^4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb" + integrity sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA== + hoist-non-react-statics@^3.0.0: version "3.3.2" resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz#ece0acaf71d62c2969c2ec59feff42a4b1a85b45" @@ -5513,6 +5540,13 @@ isarray@^2.0.5: resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== +isemail@3.x.x: + version "3.2.0" + resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.2.0.tgz#59310a021931a9fb06bbb51e155ce0b3f236832c" + integrity sha512-zKqkK+O+dGqevc93KNsbZ/TqTUFd46MwWjYOoMrjIMZ51eU7DtQG3Wmd9SQQT7i7RVnuTPEiYEWHU3MSbxC1Tg== + dependencies: + punycode "2.x.x" + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -6344,6 +6378,15 @@ jest@^24.9.0: import-local "^2.0.0" jest-cli "^24.9.0" +joi@^13.1.2: + version "13.7.0" + resolved "https://registry.yarnpkg.com/joi/-/joi-13.7.0.tgz#cfd85ebfe67e8a1900432400b4d03bbd93fb879f" + integrity sha512-xuY5VkHfeOYK3Hdi91ulocfuFopwgbSORmIwzcwHKESQhC7w1kD5jaVSPnqDxS2I8t3RZ9omCKAxNwXN5zG1/Q== + dependencies: + hoek "5.x.x" + isemail "3.x.x" + topo "3.x.x" + js-base64@^2.3.2: version "2.6.4" resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.6.4.tgz#f4e686c5de1ea1f867dbcad3d46d969428df98c4" @@ -7078,7 +7121,7 @@ mobx@^6.0.1: resolved "https://registry.yarnpkg.com/mobx/-/mobx-6.0.4.tgz#8fc3e3629a3346f8afddf5bd954411974744dad1" integrity sha512-wT2QJT9tW19VSHo9x7RPKU3z/I2Ps6wUS8Kb1OO+kzmg7UY3n4AkcaYG6jq95Lp1R9ohjC/NGYuT2PtuvBjhFg== -moment-timezone@^0.5.21: +moment-timezone@^0.5.21, moment-timezone@^0.5.x: version "0.5.32" resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.32.tgz#db7677cc3cc680fd30303ebd90b0da1ca0dfecc2" integrity sha512-Z8QNyuQHQAmWucp8Knmgei8YNo28aLjJq6Ma+jy1ZSpSk5nyfRT8xgUbSQvD2+2UajISfenndwvFuH3NGS+nvA== @@ -7135,7 +7178,7 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" -nan@^2.12.1, nan@^2.14.1: +nan@^2.12.1, nan@^2.13.2, nan@^2.14.1: version "2.14.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== @@ -7230,6 +7273,14 @@ node-environment-flags@^1.0.5: object.getownpropertydescriptors "^2.0.3" semver "^5.7.0" +node-expat@^2.3.18: + version "2.3.18" + resolved "https://registry.yarnpkg.com/node-expat/-/node-expat-2.3.18.tgz#d9e6949cecda15e131f14259b73dc7b9ed7bc560" + integrity sha512-9dIrDxXePa9HSn+hhlAg1wXkvqOjxefEbMclGxk2cEnq/Y3U7Qo5HNNqeo3fQ4bVmLhcdt3YN1TZy7WMZy4MHw== + dependencies: + bindings "^1.5.0" + nan "^2.13.2" + node-fetch-h2@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/node-fetch-h2/-/node-fetch-h2-2.3.0.tgz#c6188325f9bd3d834020bf0f2d6dc17ced2241ac" @@ -8223,16 +8274,16 @@ punycode@1.3.2: resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= +punycode@2.x.x, punycode@^2.1.0, punycode@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + punycode@^1.2.4: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= -punycode@^2.1.0, punycode@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== - pupa@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/pupa/-/pupa-2.1.1.tgz#f5e8fd4afc2c5d97828faa523549ed8744a20d62" @@ -9906,6 +9957,13 @@ toidentifier@1.0.0: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.0.tgz#7e1be3470f1e77948bc43d94a3c8f4d7752ba553" integrity sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw== +topo@3.x.x: + version "3.0.3" + resolved "https://registry.yarnpkg.com/topo/-/topo-3.0.3.tgz#d5a67fb2e69307ebeeb08402ec2a2a6f5f7ad95c" + integrity sha512-IgpPtvD4kjrJ7CRA3ov2FhWQADwv+Tdqbsf1ZnPUSAtCJ9e1Z44MmoSGDXGk4IppoZA7jd/QRkNddlLJWlUZsQ== + dependencies: + hoek "6.x.x" + toposort-class@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toposort-class/-/toposort-class-1.0.1.tgz#7ffd1f78c8be28c3ba45cd4e1a3f5ee193bd9988" @@ -10591,6 +10649,15 @@ xml2js@^0.4.17: sax ">=0.6.0" xmlbuilder "~11.0.0" +xml2json@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/xml2json/-/xml2json-0.12.0.tgz#b2ae450b267033b76d896f86e022fa7bff678572" + integrity sha512-EPJHRWJnJUYbJlzR4pBhZODwWdi2IaYGtDdteJi0JpZ4OD31IplWALuit8r73dJuM4iHZdDVKY1tLqY2UICejg== + dependencies: + hoek "^4.2.1" + joi "^13.1.2" + node-expat "^2.3.18" + xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5"