From 843e77c390125da0d27ec5746d35b9c206db861a Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 29 May 2019 16:44:10 -0700 Subject: [PATCH] ACRFD-4: Reason Code support, misc Updates, update dependencies, update readme, update jenkinsfiles and cronjobs. --- Jenkinsfile | 39 +++-- Jenkinsfile - develop | 39 +++-- Jenkinsfile - hotfix | 39 +++-- README.md | 33 +++- api/controllers/application.js | 25 ++- api/helpers/models/application.js | 1 + api/helpers/models/feature.js | 1 + api/helpers/ttlsUtils.js | 143 +++++++++-------- api/swagger/swagger.yaml | 28 ++++ data_migration/README.md | 2 +- database.json | 72 +++++++++ openshift/templates/jobs/database-backup.yaml | 2 +- openshift/templates/jobs/files-backup.yaml | 2 +- openshift/templates/jobs/updateShapes.yaml | 20 ++- package.json | 16 +- seed/shapesMigration/updateShapes.js | 148 +++++++++--------- 16 files changed, 416 insertions(+), 194 deletions(-) create mode 100644 database.json diff --git a/Jenkinsfile b/Jenkinsfile index cfe9f91..49e9074 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,33 +36,50 @@ pipeline { options { skipDefaultCheckout() } + environment { + // this credential needs to exist in Jenkins (https://jenkins.io/doc/book/using/using-credentials) + // and should contain the RocketChat Integration Token (https://rocket.chat/docs/administrator-guides/integrations/) + ROCKETCHAT_WEBHOOK_TOKEN = credentials('rocketchat_incoming_webhook_token') + } stages { stage('Building: api (master branch)') { steps { script { try { - echo "Building: ${env.JOB_NAME} #${env.BUILD_ID}" - notifyBuild("Building: ${env.JOB_NAME} #${env.BUILD_ID}", "YELLOW") + notifyBuild("Building: ${env.JOB_NAME} ${env.BUILD_ID}", "YELLOW") + echo "Building: env.JOB_NAME=${env.JOB_NAME} env.BUILD_ID=${env.BUILD_ID}" openshiftBuild bldCfg: 'nrts-prc-api-master', showBuildLogs: 'true' } catch (e) { - notifyBuild("BUILD ${env.JOB_NAME} #${env.BUILD_ID} ABORTED", "RED") - error('Stopping early…') + notifyBuild("BUILD ${env.JOB_NAME} ${env.BUILD_ID} FAILED", "RED") + error("Building: Failed: ${e}") } + notifyBuild("Built ${env.JOB_NAME} ${env.BUILD_ID}", "GREEN") + echo "Building: Success" } } } } } -def notifyBuild(String msg = '', String colour = 'GREEN') { - if (colour == 'YELLOW') { - colorCode = '#FFFF00' - } else if (colour == 'GREEN') { +def notifyBuild(String msg = '', String colour = '') { + if (colour == 'GREEN') { colorCode = '#00FF00' - } else { + } else if (colour == 'YELLOW') { + colorCode = '#FFFF00' + } else if (colour == 'RED') { colorCode = '#FF0000' + } else { + colorCode = '#000000' // black } - // Send notifications - slackSend (color: colorCode, message: msg) + String rocketChatMessage = "{ \"attachments\": [{ \"text\":\"${msg}\", \"color\":\"${colorCode}\" }] }" + + String rocketChatWebHookURL = "https://chat.pathfinder.gov.bc.ca/hooks/${ROCKETCHAT_WEBHOOK_TOKEN}" + + // Send notifications to RocketChat + try { + sh "curl -X POST -H 'Content-type: application/json' --data '${rocketChatMessage}' ${rocketChatWebHookURL}" + } catch (e) { + echo "Notify: Failed: ${e}" + } } diff --git a/Jenkinsfile - develop b/Jenkinsfile - develop index 2ffcf5a..8e2bef0 100644 --- a/Jenkinsfile - develop +++ b/Jenkinsfile - develop @@ -3,33 +3,50 @@ pipeline { options { skipDefaultCheckout() } + environment { + // this credential needs to exist in Jenkins (https://jenkins.io/doc/book/using/using-credentials) + // and should contain the RocketChat Integration Token (https://rocket.chat/docs/administrator-guides/integrations/) + ROCKETCHAT_WEBHOOK_TOKEN = credentials('rocketchat_incoming_webhook_token') + } stages { stage('Building: api (develop branch)') { steps { script { try { - echo "Building: ${env.JOB_NAME} #${env.BUILD_ID}" - notifyBuild("Building: ${env.JOB_NAME} #${env.BUILD_ID}", "YELLOW") + notifyBuild("Building: ${env.JOB_NAME} ${env.BUILD_ID}", "YELLOW") + echo "Building: env.JOB_NAME=${env.JOB_NAME} env.BUILD_ID=${env.BUILD_ID}" openshiftBuild bldCfg: 'nrts-prc-api', showBuildLogs: 'true' } catch (e) { - notifyBuild("BUILD ${env.JOB_NAME} #${env.BUILD_ID} ABORTED", "RED") - error('Stopping early…') + notifyBuild("BUILD ${env.JOB_NAME} ${env.BUILD_ID} FAILED", "RED") + error("Building: Failed: ${e}") } + notifyBuild("Built ${env.JOB_NAME} ${env.BUILD_ID}", "GREEN") + echo "Building: Success" } } } } } -def notifyBuild(String msg = '', String colour = 'GREEN') { - if (colour == 'YELLOW') { - colorCode = '#FFFF00' - } else if (colour == 'GREEN') { +def notifyBuild(String msg = '', String colour = '') { + if (colour == 'GREEN') { colorCode = '#00FF00' - } else { + } else if (colour == 'YELLOW') { + colorCode = '#FFFF00' + } else if (colour == 'RED') { colorCode = '#FF0000' + } else { + colorCode = '#000000' // black } - // Send notifications - slackSend (color: colorCode, message: msg) + String rocketChatMessage = "{ \"attachments\": [{ \"text\":\"${msg}\", \"color\":\"${colorCode}\" }] }" + + String rocketChatWebHookURL = "https://chat.pathfinder.gov.bc.ca/hooks/${ROCKETCHAT_WEBHOOK_TOKEN}" + + // Send notifications to RocketChat + try { + sh "curl -X POST -H 'Content-type: application/json' --data '${rocketChatMessage}' ${rocketChatWebHookURL}" + } catch (e) { + echo "Notify: Failed: ${e}" + } } diff --git a/Jenkinsfile - hotfix b/Jenkinsfile - hotfix index f01b430..fa2550f 100644 --- a/Jenkinsfile - hotfix +++ b/Jenkinsfile - hotfix @@ -3,33 +3,50 @@ pipeline { options { skipDefaultCheckout() } + environment { + // this credential needs to exist in Jenkins (https://jenkins.io/doc/book/using/using-credentials) + // and should contain the RocketChat Integration Token (https://rocket.chat/docs/administrator-guides/integrations/) + ROCKETCHAT_WEBHOOK_TOKEN = credentials('rocketchat_incoming_webhook_token') + } stages { stage('Building: api (hotfix branch)') { steps { script { try { - echo "Building: ${env.JOB_NAME} #${env.BUILD_ID}" - notifyBuild("Building: ${env.JOB_NAME} #${env.BUILD_ID}", "YELLOW") + notifyBuild("Building: ${env.JOB_NAME} ${env.BUILD_ID}", "YELLOW") + echo "Building: env.JOB_NAME=${env.JOB_NAME} env.BUILD_ID=${env.BUILD_ID}" openshiftBuild bldCfg: 'nrts-prc-api-hotfix', showBuildLogs: 'true' } catch (e) { - notifyBuild("BUILD ${env.JOB_NAME} #${env.BUILD_ID} ABORTED", "RED") - error('Stopping early…') + notifyBuild("BUILD ${env.JOB_NAME} ${env.BUILD_ID} FAILED", "RED") + error("Building: Failed: ${e}") } + notifyBuild("Built ${env.JOB_NAME} ${env.BUILD_ID}", "GREEN") + echo "Building: Success" } } } } } -def notifyBuild(String msg = '', String colour = 'GREEN') { - if (colour == 'YELLOW') { - colorCode = '#FFFF00' - } else if (colour == 'GREEN') { +def notifyBuild(String msg = '', String colour = '') { + if (colour == 'GREEN') { colorCode = '#00FF00' - } else { + } else if (colour == 'YELLOW') { + colorCode = '#FFFF00' + } else if (colour == 'RED') { colorCode = '#FF0000' + } else { + colorCode = '#000000' // black } - // Send notifications - slackSend (color: colorCode, message: msg) + String rocketChatMessage = "{ \"attachments\": [{ \"text\":\"${msg}\", \"color\":\"${colorCode}\" }] }" + + String rocketChatWebHookURL = "https://chat.pathfinder.gov.bc.ca/hooks/${ROCKETCHAT_WEBHOOK_TOKEN}" + + // Send notifications to RocketChat + try { + sh "curl -X POST -H 'Content-type: application/json' --data '${rocketChatMessage}' ${rocketChatWebHookURL}" + } catch (e) { + echo "Notify: Failed: ${e}" + } } diff --git a/README.md b/README.md index 170f0c3..f1734d6 100644 --- a/README.md +++ b/README.md @@ -89,12 +89,41 @@ npm run lint-fix The API is defined in `swagger.yaml`. -If the this nrts-prc-api is running locally, you can view the api docs at: `http://localhost:3000/api/docs/` +If this project is running locally, you can view the api docs at: `http://localhost:3000/api/docs/` -This project uses npm package `swagger-tools` via `./app.js` to automatically generate the express server and its routes. +This project uses npm package `swagger-tools` via `./app.js` to automatically generate the express server and its routes, based on the contents of `swagger.yaml`. + +Useful Note: The handler function for each route is specified by the `operationId` field. Recommend reviewing the [Open API Specification](https://swagger.io/docs/specification/about/) before making any changes to the `swagger.yaml` file. +# Logging + +A centralized logger has been created (see `api/helpers/logger.js`). + +## Logger configuration +The loggers log level can be configured via an environment variable: `LOG_LEVEL` + +Set this variable to one of: `error`, `warn`, `info`, `debug` + +Default value: `info` + +## Instantiating the logger in your class/file +``` +const log = require('./logger)('a meaningful label, typically the class name`) +``` + +## Using the logger +``` +log.error('Used when logging unexpected errors. Generally these will only exist in catch() blocks'); + +log.warn('Used when logging soft errors. For example, if your request finished but returned a 404 not found'); + +log.info('General log messages about the state of the application'); + +log.debug('Useful for logging objects and other developer data', JSON.stringify(myObject)); +``` + # Testing ## Info diff --git a/api/controllers/application.js b/api/controllers/application.js index ca013d8..cebaadf 100644 --- a/api/controllers/application.js +++ b/api/controllers/application.js @@ -22,6 +22,7 @@ var tagList = [ 'publishDate', 'purpose', 'status', + 'reason', 'subpurpose', 'subtype', 'tantalisID', @@ -353,6 +354,7 @@ exports.protectedPost = function(args, res, next) { delete obj.type; delete obj.subtype; delete obj.status; + delete obj.reason; delete obj.tenureStage; delete obj.location; delete obj.businessUnit; @@ -386,6 +388,7 @@ exports.protectedPost = function(args, res, next) { savedApp.type = data.TENURE_TYPE; savedApp.subtype = data.TENURE_SUBTYPE; savedApp.status = data.TENURE_STATUS; + savedApp.reason = data.TENURE_REASON; savedApp.tenureStage = data.TENURE_STAGE; savedApp.location = data.TENURE_LOCATION; savedApp.businessUnit = data.RESPONSIBLE_BUSINESS_UNIT; @@ -524,10 +527,11 @@ exports.protectedRefresh = function(args, res, next) { var Application = require('mongoose').model('Application'); Application.findOne({ _id: objId }, function(err, applicationObject) { if (applicationObject) { - defaultLog.debug('applicationObject:', JSON.stringify(applicationObject)); + defaultLog.debug('application before refresh:', JSON.stringify(applicationObject)); TTLSUtils.updateApplication(applicationObject).then( updatedApplicationAndFeatures => { + defaultLog.debug('application after refresh:', JSON.stringify(applicationObject)); return Actions.sendResponse(res, 200, updatedApplicationAndFeatures); }, error => { @@ -692,6 +696,25 @@ var addStandardQueryFilters = function(query, args) { } _.assignIn(query, { status: { $in: queryArray } }); } + if (args.swagger.params.reason && args.swagger.params.reason.value !== undefined) { + var queryString = qs.parse(args.swagger.params.reason.value); + var queryArray = []; + if (queryString.eq) { + if (Array.isArray(queryString.eq)) { + queryArray = queryString.eq; + } else { + queryArray.push(queryString.eq); + } + _.assignIn(query, { reason: { $in: queryArray } }); + } else if (queryString.ne) { + if (Array.isArray(queryString.ne)) { + queryArray = queryString.ne; + } else { + queryArray.push(queryString.ne); + } + _.assignIn(query, { reason: { $nin: queryArray } }); + } + } if (args.swagger.params.agency && args.swagger.params.agency.value !== undefined) { _.assignIn(query, { agency: args.swagger.params.agency.value }); } diff --git a/api/helpers/models/application.js b/api/helpers/models/application.js index e73580b..88023f5 100644 --- a/api/helpers/models/application.js +++ b/api/helpers/models/application.js @@ -18,6 +18,7 @@ module.exports = require('../models')('Application', { publishDate: { type: Date }, purpose: { type: String }, status: { type: String }, + reason: { type: String }, subpurpose: { type: String }, subtype: { type: String }, tantalisID: { type: Number, default: 0 }, diff --git a/api/helpers/models/feature.js b/api/helpers/models/feature.js index c898da9..4b8a316 100644 --- a/api/helpers/models/feature.js +++ b/api/helpers/models/feature.js @@ -13,6 +13,7 @@ module.exports = require('../models')('Feature', { INTRID_SID: { type: Number, default: 0 }, TENURE_STAGE: { type: String, default: '' }, TENURE_STATUS: { type: String, default: '' }, + TENURE_REASON: { type: String, default: '' }, TENURE_TYPE: { type: String, default: '' }, TENURE_SUBTYPE: { type: String, default: '' }, TENURE_PURPOSE: { type: String, default: '' }, diff --git a/api/helpers/ttlsUtils.js b/api/helpers/ttlsUtils.js index 291ad51..8b4fa19 100644 --- a/api/helpers/ttlsUtils.js +++ b/api/helpers/ttlsUtils.js @@ -13,18 +13,28 @@ const helpers = require('@turf/helpers'); const spatialUtils = require('./spatialUtils'); const defaultLog = require('./logger')('ttlsUtils'); -let tantalisAPI = process.env.TTLS_API_ENDPOINT || 'https://api.nrs.gov.bc.ca/ttls-api/v1/'; -let webADEAPI = process.env.WEBADE_AUTH_ENDPOINT || 'https://api.nrs.gov.bc.ca/oauth2/v1/'; -let username = process.env.WEBADE_USERNAME || 'TTLS-EXT'; +let tantalisAPI = + process.env.TTLS_API_ENDPOINT || + 'https://t1api.nrs.gov.bc.ca/ttls-api/v1/' || + 'https://api.nrs.gov.bc.ca/ttls-api/v1/'; +let webADEAPI = + process.env.WEBADE_AUTH_ENDPOINT || + 'https://t1api.nrs.gov.bc.ca/oauth2/v1/' || + 'https://api.nrs.gov.bc.ca/oauth2/v1/'; +let username = process.env.WEBADE_USERNAME || 'ACRFD_SERVICE_CLIENT' || 'TTLS-EXT'; let password = process.env.WEBADE_PASSWORD; // WebADE Login exports.loginWebADE = function() { // Login to webADE and return access_token for use in subsequent calls. return new Promise(function(resolve, reject) { + const url = webADEAPI + 'oauth/token?grant_type=client_credentials&disableDeveloperFilter=true&scope=TTLS.*'; + + defaultLog.debug('WebADE Login url:', url); + request.get( { - url: webADEAPI + 'oauth/token?grant_type=client_credentials&disableDeveloperFilter=true&scope=TTLS.*', + url, headers: { Authorization: 'Basic ' + Buffer.from(username + ':' + password).toString('base64') } @@ -34,7 +44,7 @@ exports.loginWebADE = function() { defaultLog.error('WebADE Login Error:', err); reject(err); } else if (res && res.statusCode !== 200) { - defaultLog.warn('WebADE Login ResponseCode:', res.statusCode); + defaultLog.warn('WebADE Login Response:', res.statusCode, body); reject({ code: (res && res.statusCode) || null }); } else { try { @@ -66,22 +76,16 @@ exports.loginWebADE = function() { */ exports.getApplicationByFilenumber = function(accessToken, clFile, pageNumber = 1, pageRowCount = 100) { return new Promise(function(resolve, reject) { - defaultLog.info( - 'Looking up file:', + const url = tantalisAPI + - 'landUseApplications' + - `?fileNumber=${clFile}` + - `&pageNumber=${pageNumber}` + - `&pageRowCount=${pageRowCount}` - ); + 'landUseApplications' + + `?fileNumber=${clFile}&pageNumber=${pageNumber}&pageRowCount=${pageRowCount}`; + + defaultLog.info('Looking up tantalis applications by crown land file number:', url); + request.get( { - url: - tantalisAPI + - 'landUseApplications' + - `?fileNumber=${clFile}` + - `&pageNumber=${pageNumber}` + - `&pageRowCount=${pageRowCount}`, + url, auth: { bearer: accessToken } @@ -91,7 +95,7 @@ exports.getApplicationByFilenumber = function(accessToken, clFile, pageNumber = defaultLog.error('TTLS API Error:', err); reject(err); } else if (res && res.statusCode !== 200) { - defaultLog.warn('TTLS API ResponseCode:', res.statusCode); + defaultLog.warn('TTLS API Response:', res.statusCode, body); reject({ code: (res && res.statusCode) || null }); } else { try { @@ -101,14 +105,23 @@ exports.getApplicationByFilenumber = function(accessToken, clFile, pageNumber = if (obj && obj.elements && obj.elements.length > 0) { for (let app of obj.elements) { var application = {}; - application.TENURE_PURPOSE = app.purposeCode['description']; - application.TENURE_SUBPURPOSE = app.purposeCode.subPurposeCodes[0]['description']; - application.TENURE_TYPE = app.landUseTypeCode['description']; - application.TENURE_SUBTYPE = app.landUseTypeCode.landUseSubTypeCodes[0]['description']; - application.TENURE_STATUS = app.statusCode['description']; - application.TENURE_STAGE = app.stageCode['description']; + application.TENURE_PURPOSE = app.purposeCode && app.purposeCode['description']; + application.TENURE_SUBPURPOSE = + app.purposeCode && + app.purposeCode.subPurposeCodes && + app.purposeCode.subPurposeCodes[0] && + app.purposeCode.subPurposeCodes[0]['description']; + application.TENURE_TYPE = app.landUseTypeCode && app.landUseTypeCode['description']; + application.TENURE_SUBTYPE = + app.landUseTypeCode && + app.landUseTypeCode.landUseSubTypeCodes && + app.landUseTypeCode.landUseSubTypeCodes[0] && + app.landUseTypeCode.landUseSubTypeCodes[0]['description']; + application.TENURE_STATUS = app.statusCode && app.statusCode['description']; + application.TENURE_REASON = app.reasonCode && app.reasonCode['description']; + application.TENURE_STAGE = app.stageCode && app.stageCode['description']; application.TENURE_LOCATION = app.locationDescription; - application.RESPONSIBLE_BUSINESS_UNIT = app.businessUnit.name; + application.RESPONSIBLE_BUSINESS_UNIT = app.businessUnit && app.businessUnit.name; application.CROWN_LANDS_FILE = app.fileNumber; application.DISPOSITION_TRANSACTION_SID = app.landUseApplicationId; applications.push(application); @@ -138,22 +151,14 @@ exports.getApplicationByFilenumber = function(accessToken, clFile, pageNumber = */ exports.getApplicationByDispositionID = function(accessToken, dispositionID, pageNumber = 1, pageRowCount = 100) { return new Promise(function(resolve, reject) { - defaultLog.info( - 'Looking up disposition:', - tantalisAPI + - 'landUseApplications/' + - dispositionID + - `&pageNumber=${pageNumber}` + - `&pageRowCount=${pageRowCount}` - ); + const url = + tantalisAPI + 'landUseApplications/' + dispositionID + `?pageNumber=${pageNumber}&pageRowCount=${pageRowCount}`; + + defaultLog.info('Looking up tantalis applications by disposition id:', url); + request.get( { - url: - tantalisAPI + - 'landUseApplications/' + - dispositionID + - `?pageNumber=${pageNumber}` + - `&pageRowCount=${pageRowCount}`, + url, auth: { bearer: accessToken } @@ -163,7 +168,7 @@ exports.getApplicationByDispositionID = function(accessToken, dispositionID, pag defaultLog.error('TTLS API Error:', err); reject(err); } else if (res && res.statusCode !== 200) { - defaultLog.warn('TTLS API ResponseCode:', res.statusCode); + defaultLog.warn('TTLS API Response:', res.statusCode, body); reject({ code: (res && res.statusCode) || null }); } else { try { @@ -172,20 +177,29 @@ exports.getApplicationByDispositionID = function(accessToken, dispositionID, pag var application = {}; if (obj) { // Setup the application object. - application.TENURE_PURPOSE = obj.purposeCode['description']; - application.TENURE_SUBPURPOSE = obj.purposeCode.subPurposeCodes[0]['description']; - application.TENURE_TYPE = obj.landUseTypeCode['description']; - application.TENURE_SUBTYPE = obj.landUseTypeCode.landUseSubTypeCodes[0]['description']; - application.TENURE_STATUS = obj.statusCode['description']; - application.TENURE_STAGE = obj.stageCode['description']; + application.TENURE_PURPOSE = obj.purposeCode && obj.purposeCode['description']; + application.TENURE_SUBPURPOSE = + obj.purposeCode && + obj.purposeCode.subPurposeCodes && + obj.purposeCode.subPurposeCodes[0] && + obj.purposeCode.subPurposeCodes[0]['description']; + application.TENURE_TYPE = obj.landUseTypeCode && obj.landUseTypeCode['description']; + application.TENURE_SUBTYPE = + obj.landUseTypeCode && + obj.landUseTypeCode.landUseSubTypeCodes && + obj.landUseTypeCode.landUseSubTypeCodes[0] && + obj.landUseTypeCode.landUseSubTypeCodes[0]['description']; + application.TENURE_STATUS = obj.statusCode && obj.statusCode['description']; + application.TENURE_REASON = obj.reasonCode && obj.reasonCode['description']; + application.TENURE_STAGE = obj.stageCode && obj.stageCode['description']; application.TENURE_LOCATION = obj.locationDescription; - application.RESPONSIBLE_BUSINESS_UNIT = obj.businessUnit.name; + application.RESPONSIBLE_BUSINESS_UNIT = obj.businessUnit && obj.businessUnit.name; application.CROWN_LANDS_FILE = obj.fileNumber; application.DISPOSITION_TRANSACTION_SID = dispositionID; application.parcels = []; application.interestedParties = []; application.statusHistoryEffectiveDate = - obj.statusHistory[0] != null + obj.statusHistory && obj.statusHistory[0] != null ? new Date(obj.statusHistory[0].effectiveDate) // convert Unix Epoch Time (ms) : null; @@ -247,12 +261,12 @@ exports.getApplicationByDispositionID = function(accessToken, dispositionID, pag partyObj.interestedPartyType = party.interestedPartyType; if (party.interestedPartyType == 'I') { - partyObj.firstName = party.individual.firstName; - partyObj.lastName = party.individual.lastName; + partyObj.firstName = party.individual && party.individual.firstName; + partyObj.lastName = party.individual && party.individual.lastName; } else { // party.interestedPartyType == 'O' - partyObj.legalName = party.organization.legalName; - partyObj.divisionBranch = party.organization.divisionBranch; + partyObj.legalName = party.organization && party.organization.legalName; + partyObj.divisionBranch = party.organization && party.organization.divisionBranch; } // Check if we've already added this. if (!_.includes(application.interestedParties, partyObj)) { @@ -313,21 +327,18 @@ const internalGetAllApplicationIDs = function( applicationIDs = [] ) { return new Promise(function(resolve, reject) { - const queryString = `?${qs.stringify(filterParams)}`; + const url = + tantalisAPI + + 'landUseApplications' + + `?${qs.stringify(filterParams)}` + + `&pageNumber=${pageNumber}` + + `&pageRowCount=${pageRowCount}`; - defaultLog.info( - 'Looking up all applications:', - tantalisAPI + 'landUseApplications' + queryString + `&pageNumber=${pageNumber}` + `&pageRowCount=${pageRowCount}` - ); + defaultLog.info('Looking up all tantalis applications:', url); request.get( { - url: - tantalisAPI + - 'landUseApplications' + - queryString + - `&pageNumber=${pageNumber}` + - `&pageRowCount=${pageRowCount}`, + url, auth: { bearer: accessToken } @@ -337,7 +348,7 @@ const internalGetAllApplicationIDs = function( defaultLog.error('TTLS API Error:', err); reject(err); } else if (res && res.statusCode !== 200) { - defaultLog.info('TTLS API ResponseCode:', res.statusCode); + defaultLog.warn('TTLS API Response:', res.statusCode, body); reject({ code: (res && res.statusCode) || null }); } else { try { @@ -456,6 +467,7 @@ const updateFeatures = function(acrfdApp, tantalisApp) { f.properties.TENURE_PURPOSE = tantalisApp.TENURE_PURPOSE; f.properties.TENURE_SUBPURPOSE = tantalisApp.TENURE_SUBPURPOSE; f.properties.TENURE_STATUS = tantalisApp.TENURE_STATUS; + f.properties.TENURE_REASON = tantalisApp.TENURE_REASON; f.properties.TENURE_TYPE = tantalisApp.TENURE_TYPE; f.properties.TENURE_STAGE = tantalisApp.TENURE_STAGE; f.properties.TENURE_SUBTYPE = tantalisApp.TENURE_SUBTYPE; @@ -543,6 +555,7 @@ const updateApplicationMeta = function(acrfdApp, tantalisApp) { updatedAppObject.purpose = tantalisApp.TENURE_PURPOSE; updatedAppObject.subpurpose = tantalisApp.TENURE_SUBPURPOSE; updatedAppObject.status = tantalisApp.TENURE_STATUS; + updatedAppObject.reason = tantalisApp.TENURE_REASON; updatedAppObject.type = tantalisApp.TENURE_TYPE; updatedAppObject.tenureStage = tantalisApp.TENURE_STAGE; updatedAppObject.subtype = tantalisApp.TENURE_SUBTYPE; diff --git a/api/swagger/swagger.yaml b/api/swagger/swagger.yaml index 2b83aa9..8c1d8cd 100644 --- a/api/swagger/swagger.yaml +++ b/api/swagger/swagger.yaml @@ -131,6 +131,9 @@ definitions: status: type: string example: "ACCEPTED" + reason: + type: string + example: "OFFER NOT ACCEPTED" subpurpose: type: string example: "MARINA" @@ -170,6 +173,7 @@ definitions: - publishDate - purpose - status + - reason - subpurpose - subtype - tantalisID @@ -847,6 +851,12 @@ paths: type: string required: false description: "Status(es) that Applications must match" + - in: query + collectionFormat: multi + name: reason + type: string + required: false + description: "Reason(s) that Applications must match" - in: query name: type type: string @@ -968,6 +978,12 @@ paths: type: string required: false description: "Status(es) that Applications must match" + - in: query + collectionFormat: multi + name: reason + type: string + required: false + description: "Reason(s) that Applications must match" - in: query name: type type: string @@ -1427,6 +1443,12 @@ paths: type: string required: false description: "Status(es) that Applications must match" + - in: query + collectionFormat: multi + name: reason + type: string + required: false + description: "Reason(s) that Applications must match" - in: query name: type type: string @@ -1556,6 +1578,12 @@ paths: type: string required: false description: "Status(es) that Applications must match" + - in: query + collectionFormat: multi + name: reason + type: string + required: false + description: "Reason(s) that Applications must match" - in: query name: type type: string diff --git a/data_migration/README.md b/data_migration/README.md index 7f9cc20..0114c08 100644 --- a/data_migration/README.md +++ b/data_migration/README.md @@ -1 +1 @@ -This folder contains database update commands that can be run manually via a mongo console or GUI. \ No newline at end of file +This folder contains database update commands that can be run manually via a mongo console or GUI. diff --git a/database.json b/database.json new file mode 100644 index 0000000..e373e27 --- /dev/null +++ b/database.json @@ -0,0 +1,72 @@ +{ + "defaultEnv": "local", + + "local": { + "driver": "mongodb", + "database": "nrts-dev", + "host": "localhost" + }, + + "dev": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_SERVICE_HOST" + }, + + "test": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_SERVICE_HOST" + }, + + "demo": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "DEMO_MONGODB_SERVICE_HOST" + }, + + "scale": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_SCALE_SERVICE_HOST" + }, + + "beta": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_BETA_SERVICE_HOST" + }, + + "master": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_MASTER_SERVICE_HOST" + }, + + "prod": { + "driver": "mongodb", + "database": { "ENV": "MONGODB_DATABASE" }, + "user": { "ENV": "MONGODB_USERNAME" }, + "password": { "ENV": "MONGODB_PASSWORD" }, + "authSource": "admin", + "host": "MONGODB_SERVICE_HOST" + } +} diff --git a/openshift/templates/jobs/database-backup.yaml b/openshift/templates/jobs/database-backup.yaml index eef9881..ffba8ab 100644 --- a/openshift/templates/jobs/database-backup.yaml +++ b/openshift/templates/jobs/database-backup.yaml @@ -24,7 +24,7 @@ spec: - name: MONGODB_SVC_HOST value: '127.0.0.1' - name: JSON_PAYLOAD - value: '{"text":"Backup Job for nrts-prc-prod Completed Successfully!"}' + value: '{"attachments":[{"text":"MongoDB Backup Job for nrts-prc-prod Completed Successfully!", "color":"#00FF00"}]}' - name: NOTIFICATION_URL value: '' image: registry.access.redhat.com/rhscl/mongodb-32-rhel7 diff --git a/openshift/templates/jobs/files-backup.yaml b/openshift/templates/jobs/files-backup.yaml index 3f358c9..1a9d44f 100644 --- a/openshift/templates/jobs/files-backup.yaml +++ b/openshift/templates/jobs/files-backup.yaml @@ -16,7 +16,7 @@ spec: image: registry.access.redhat.com/rhscl/mongodb-32-rhel7 env: - name: JSON_PAYLOAD - value: '{"text":"File Backup Job Completed Successfully!"}' + value: '{"attachments":[{"text":"File Backup Job for nrts-prc-prod Completed Successfully!", "color":"#00FF00"}]}' - name: NOTIFICATION_URL value: '' volumeMounts: diff --git a/openshift/templates/jobs/updateShapes.yaml b/openshift/templates/jobs/updateShapes.yaml index 6a246b8..5aa4941 100644 --- a/openshift/templates/jobs/updateShapes.yaml +++ b/openshift/templates/jobs/updateShapes.yaml @@ -1,4 +1,4 @@ -apiVersion: batch/v2alpha1 +apiVersion: batch/v1beta1 kind: CronJob metadata: labels: @@ -34,21 +34,29 @@ spec: - name: AUTH_ENDPOINT value: '' - name: JSON_PAYLOAD - value: '{"text":"Shapes Import Job for nrts-prc-prod Completed Successfully!"}' + value: '{"attachments":[{"text":"Shapes Import Job for nrts-prc-prod Completed Successfully!", "color":"#00FF00"}]}' - name: JSON_PAYLOAD_FAIL - value: '{"text":"Shapes Import Job for nrts-prc-prod Failed!"}' + value: '{"attachments":[{"text":"Shapes Import Job for nrts-prc-prod Failed!", "color":"#FF0000"}]}' - name: NOTIFICATION_URL value: '' + - name: WEBADE_AUTH_ENDPOINT + value: '' + - name: WEBADE_USERNAME + value: '' + - name: WEBADE_PASSWORD + value: '' + - name: TTLS_API_ENDPOINT + value: '' image: docker-registry.default.svc:5000/nrts-prc-tools/nrts-prc-api:master command: - bash - -c - 'npm install --prefix seed/; - node seed/shapesMigration/update.js "${API_USERNAME}" "${API_PASSWORD}" "${API_PROTOCOL}" "${API_HOST}" "${API_PORT}" "${CLIENT_ID}" "${GRANT_TYPE}" "${AUTH_ENDPOINT}"; + node seed/shapesMigration/updateShapes.js "${API_USERNAME}" "${API_PASSWORD}" "${API_PROTOCOL}" "${API_HOST}" "${API_PORT}" "${CLIENT_ID}" "${GRANT_TYPE}" "${AUTH_ENDPOINT}"; if [ "${PIPESTATUS[0]}" -ne "1" ]; then curl -X POST -H "Content-type: application/json" --data "${JSON_PAYLOAD}" "${NOTIFICATION_URL}"; - else curl -X POST -H "Content-type: application/json" --data "${JSON_PAYLOAD_FAIL}" "${NOTIFICATION_URL}" - fi' + else curl -X POST -H "Content-type: application/json" --data "${JSON_PAYLOAD_FAIL}" "${NOTIFICATION_URL}"; + fi;' imagePullPolicy: Always resources: {} terminationMessagePath: /dev/termination-log diff --git a/package.json b/package.json index 2c9a9a2..968968d 100644 --- a/package.json +++ b/package.json @@ -46,24 +46,24 @@ "swagger-tools": "0.10.4", "underscore": "1.9.1", "validator": "10.11.0", - "wkx": "0.4.6", + "wkx": "0.4.7", "winston": "2.4.4", "yamljs": "0.3.0" }, "devDependencies": { - "babel-eslint": "10.0.1", + "babel-eslint": "10.0.2", "database-cleaner": "1.3.0", - "eslint": "5.16.0", - "eslint-config-prettier": "4.3.0", + "eslint": "6.0.1", + "eslint-config-prettier": "6.0.0", "factory-girl": "5.0.4", - "husky": "2.3.0", + "husky": "3.0.0", "jest": "24.8.0", "jest-config": "24.8.0", - "lint-staged": "8.1.7", - "mongodb-memory-server": "5.1.2", + "lint-staged": "9.2.0", + "mongodb-memory-server": "5.1.5", "nock": "10.0.6", "npm-run-all": "4.1.5", - "prettier": "1.17.1", + "prettier": "1.18.2", "shelljs": "0.8.3", "supertest": "4.0.2" }, diff --git a/seed/shapesMigration/updateShapes.js b/seed/shapesMigration/updateShapes.js index f7529a8..7902872 100644 --- a/seed/shapesMigration/updateShapes.js +++ b/seed/shapesMigration/updateShapes.js @@ -15,26 +15,26 @@ // winston logger needs to be created before any local classes that use the logger are loaded. const defaultLog = require('../../api/helpers/logger')('updateShapes'); -var Promise = require('es6-promise').Promise; -var _ = require('lodash'); -var request = require('request'); -var querystring = require('querystring'); -var moment = require('moment'); -var TTLSUtils = require('../../api/helpers/ttlsUtils'); -var Actions = require('../../api/helpers/actions'); +const Promise = require('es6-promise').Promise; +const _ = require('lodash'); +const request = require('request'); +const querystring = require('querystring'); +const moment = require('moment'); +const TTLSUtils = require('../../api/helpers/ttlsUtils'); +const Actions = require('../../api/helpers/actions'); -var username = ''; -var password = ''; -var protocol = 'http'; -var host = 'localhost'; -var port = '3000'; -var uri = ''; -var client_id = ''; -var grant_type = ''; -var auth_endpoint = 'http://localhost:3000/api/login/token'; -var _accessToken = ''; +let username = ''; +let password = ''; +let protocol = 'http'; +let host = 'localhost'; +let port = '3000'; +let uri = ''; +let client_id = ''; +let grant_type = ''; +let auth_endpoint = 'http://localhost:3000/api/login/token'; +let _accessToken = ''; -var args = process.argv.slice(2); +const args = process.argv.slice(2); defaultLog.info('======================================================='); if (args.length !== 8) { defaultLog.error( @@ -59,7 +59,7 @@ if (args.length !== 8) { } // Used when unpublishing retired applications. -var retiredStatuses = [ +const retiredStatuses = [ 'ABANDONED', 'CANCELLED', 'OFFER NOT ACCEPTED', @@ -78,9 +78,9 @@ var retiredStatuses = [ ]; // Used to renew the ACRFD login tokes before it expires if the update script takes longer than the lifespan of the token. -var jwt_login = null; // the ACRFD login token -var jwt_expiry = null; // how long the token lasts before expiring -var jwt_login_time = null; // time we last logged in +let jwt_login = null; // the ACRFD login token +let jwt_expiry = null; // how long the token lasts before expiring +let jwt_login_time = null; // time we last logged in /** * Logs in to ACRFD. @@ -89,15 +89,15 @@ var jwt_login_time = null; // time we last logged in * @param {String} password * @returns {Promise} promise that resolves with the jwt_login token. */ -var loginToACRFD = function(username, password) { - return new Promise(function(resolve, reject) { - var body = querystring.stringify({ +const loginToACRFD = function(username, password) { + return new Promise((resolve, reject) => { + const body = querystring.stringify({ grant_type: grant_type, client_id: client_id, username: username, password: password }); - var contentLength = body.length; + const contentLength = body.length; request.post( { url: auth_endpoint, @@ -107,15 +107,15 @@ var loginToACRFD = function(username, password) { }, body: body }, - function(error, res, body) { + (error, res, body) => { if (error) { defaultLog.error(' - loginToACRFD error:', error); reject(error); } else if (res.statusCode !== 200) { - defaultLog.error(' - loginToACRFD error:', res.statusCode, body); + defaultLog.warn(' - loginToACRFD response:', res.statusCode, body); reject(res.statusCode + ' ' + body); } else { - var data = JSON.parse(body); + const data = JSON.parse(body); jwt_login = data.access_token; jwt_expiry = data.expires_in; jwt_login_time = moment(); @@ -131,15 +131,13 @@ var loginToACRFD = function(username, password) { * * @returns {Promise} */ -var renewJWTLogin = function() { - return new Promise(function(resolve, reject) { - var duration = moment.duration(moment().diff(jwt_login_time)).asSeconds(); +const renewJWTLogin = function() { + return new Promise((resolve, reject) => { + const duration = moment.duration(moment().diff(jwt_login_time)).asSeconds(); // if less than 60 seconds left before token expiry. if (duration > jwt_expiry - 60) { defaultLog.info(' - Requesting new ACRFD login token.'); - return loginToACRFD(username, password).then(function() { - resolve(); - }); + return loginToACRFD(username, password).then(() => resolve()); } else { resolve(); } @@ -151,13 +149,13 @@ var renewJWTLogin = function() { * * @returns {Promise} promise that resolves with the list of retired applications. */ -var getApplicationsToUnpublish = function() { +const getApplicationsToUnpublish = function() { defaultLog.info(' - fetching retired applications.'); - return new Promise(function(resolve, reject) { - var untilDate = moment().subtract(6, 'months'); + return new Promise((resolve, reject) => { + const untilDate = moment().subtract(6, 'months'); // get all applications that are in a retired status and that have a last status update date older than 6 months ago. - var queryString = `?statusHistoryEffectiveDate[until]=${untilDate.toISOString()}`; + let queryString = `?statusHistoryEffectiveDate[until]=${untilDate.toISOString()}`; retiredStatuses.forEach(status => (queryString += `&status[eq]=${encodeURIComponent(status)}`)); request.get( @@ -168,18 +166,18 @@ var getApplicationsToUnpublish = function() { Authorization: 'Bearer ' + jwt_login } }, - function(error, res, body) { + (error, res, body) => { if (error) { defaultLog.error(' - getApplicationsToUnpublish error:', error); reject(error); } else if (res.statusCode !== 200) { - defaultLog.error(' - getApplicationsToUnpublish error:', res.statusCode, body); + defaultLog.warn(' - getApplicationsToUnpublish response:', res.statusCode, body); reject(res.statusCode + ' ' + body); } else { - var data = JSON.parse(body); + const data = JSON.parse(body); // only return applications that are currently published - var appsToUnpublish = _.filter(data, app => { + const appsToUnpublish = _.filter(data, app => { return Actions.isPublished(app); }); resolve(appsToUnpublish); @@ -195,10 +193,10 @@ var getApplicationsToUnpublish = function() { * @param {*} applicationsToUnpublish array of applications * @returns {Promise} */ -var unpublishApplications = function(applicationsToUnpublish) { - return applicationsToUnpublish.reduce(function(previousApp, currentApp) { - return previousApp.then(function() { - return new Promise(function(resolve, reject) { +const unpublishApplications = function(applicationsToUnpublish) { + return applicationsToUnpublish.reduce((previousApp, currentApp) => { + return previousApp.then(() => { + return new Promise((resolve, reject) => { request.put( { url: uri + 'api/application/' + currentApp._id + '/unpublish', @@ -208,16 +206,16 @@ var unpublishApplications = function(applicationsToUnpublish) { }, body: JSON.stringify(currentApp) }, - function(error, res, body) { + (error, res, body) => { if (error) { defaultLog.error(' - unpublishApplications error:', error); reject(error); } else if (res.statusCode !== 200) { - defaultLog.error(' - unpublishApplications error:', res.statusCode, body); + defaultLog.warn(' - unpublishApplications response:', res.statusCode, body); reject(res.statusCode + ' ' + body); } else { defaultLog.info(` - Unpublished application, _id: ${currentApp._id}`); - var data = JSON.parse(body); + const data = JSON.parse(body); resolve(data); } } @@ -234,7 +232,7 @@ var unpublishApplications = function(applicationsToUnpublish) { * @returns {Promise} */ const updateACRFDApplication = function(acrfdAppID) { - return new Promise(function(resolve, reject) { + return new Promise((resolve, reject) => { // only update the ones that aren't deleted const url = uri + `api/application/${acrfdAppID}/refresh`; request.put( @@ -245,15 +243,15 @@ const updateACRFDApplication = function(acrfdAppID) { Authorization: 'Bearer ' + jwt_login } }, - function(error, res, body) { + (error, res, body) => { if (error) { defaultLog.error(' - updateACRFDApplication error:', error); reject(error); } else if (res.statusCode !== 200) { - defaultLog.error(' - updateACRFDApplication error:', res.statusCode, body); + defaultLog.warn(' - updateACRFDApplication response:', res.statusCode, body); reject(res.statusCode + ' ' + body); } else { - var obj = {}; + let obj = {}; try { obj = JSON.parse(body); resolve(obj); @@ -273,8 +271,8 @@ const updateACRFDApplication = function(acrfdAppID) { * * @returns {Promise} promise that resolves with an array of ACRFD applications. */ -var getAllACRFDApplicationIDs = function() { - return new Promise(function(resolve, reject) { +const getAllACRFDApplicationIDs = function() { + return new Promise((resolve, reject) => { // only update the ones that aren't deleted const url = uri + 'api/application/' + '?fields=tantalisID&isDeleted=false'; request.get( @@ -285,15 +283,15 @@ var getAllACRFDApplicationIDs = function() { Authorization: 'Bearer ' + jwt_login } }, - function(error, res, body) { + (error, res, body) => { if (error) { defaultLog.error(' - getAllACRFDApplicationIDs error:', error); reject(error); } else if (res.statusCode !== 200) { - defaultLog.error(' - getAllACRFDApplicationIDs error:', res.statusCode, body); + defaultLog.warn(' - getAllACRFDApplicationIDs response:', res.statusCode, body); reject(res.statusCode + ' ' + body); } else { - var obj = {}; + let obj = {}; try { obj = JSON.parse(body); resolve(obj); @@ -311,68 +309,66 @@ var getAllACRFDApplicationIDs = function() { */ defaultLog.info('1. Authenticating with ACRFD.'); loginToACRFD(username, password) - .then(function() { + .then(() => { defaultLog.info('-----------------------------------------------'); defaultLog.info('2. Unpublishing retired applications.'); - return getApplicationsToUnpublish().then(function(applicationsToUnpublish) { + return getApplicationsToUnpublish().then(applicationsToUnpublish => { defaultLog.info(` - found ${applicationsToUnpublish.length} retired applications.`); return unpublishApplications(applicationsToUnpublish); }); }) - .then(function() { + .then(() => { defaultLog.info('-----------------------------------------------'); defaultLog.info('3. Authenticating with Tantalis.'); - return TTLSUtils.loginWebADE().then(function(accessToken) { + return TTLSUtils.loginWebADE().then(accessToken => { defaultLog.info(' - TTLS API login token:', accessToken); _accessToken = accessToken; return _accessToken; }); }) - .then(function() { + .then(() => { defaultLog.info('-----------------------------------------------'); defaultLog.info( '4. Fetching all Tantalis applications that have had their status history effective date updated in the last week.' ); - var lastWeek = moment() + const lastWeek = moment() .subtract(1, 'week') .format('YYYYMMDD'); return TTLSUtils.getAllApplicationIDs(_accessToken, { updated: lastWeek }); }) - .then(function(recentlyUpdatedApplicationIDs) { + .then(recentlyUpdatedApplicationIDs => { defaultLog.info('-----------------------------------------------'); defaultLog.info( '5. Fetching all non-deleted ACRFD applications and cross referencing with recently updated Tantalis applications.' ); - return getAllACRFDApplicationIDs().then(function(allACRFDApplicationIDs) { + return getAllACRFDApplicationIDs().then(allACRFDApplicationIDs => { return allACRFDApplicationIDs .filter(acrfdApp => recentlyUpdatedApplicationIDs.includes(acrfdApp.tantalisID)) .map(acrfdApp => acrfdApp._id); }); }) - .then(function(applicationIDsToUpdate) { + .then(applicationIDsToUpdate => { defaultLog.info( - ` - Found ${ - applicationIDsToUpdate.length - } ACRFD Applications with a matching recently updated Tantalis application.` + ` - Found ${applicationIDsToUpdate.length} ACRFD Applications with a matching recently updated Tantalis application.` ); // For each ACRFD application with a matching recently updated application from Tantalis, fetch the matching record in ACRFD and update it - return applicationIDsToUpdate.reduce(function(previousItem, currentItem) { - return previousItem.then(function() { + return applicationIDsToUpdate.reduce((previousItem, currentItem) => { + return previousItem.then(() => { defaultLog.info('-----------------------------------------------'); defaultLog.info(`6. Updating ACRFD Application, _id: ${currentItem}`); // Each iteration, check if the ACRFD login token is nearly expired and needs to be re-fetched - return renewJWTLogin().then(function() { + return renewJWTLogin().then(() => { return updateACRFDApplication(currentItem); }); }); }, Promise.resolve()); }) - .then(function() { + .then(() => { defaultLog.info('-----------------------------------------------'); defaultLog.info('Done!'); defaultLog.info('======================================================='); }) - .catch(function(error) { + .catch(error => { defaultLog.error('-----------------------------------------------'); defaultLog.error(' - General error:', error); defaultLog.error('=======================================================');