From 3b39468d7446c678a882eec42e87c56e86ae6e65 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 4 Jul 2024 09:44:53 +0300 Subject: [PATCH 1/4] paginate oscar referrals --- jobs/V2/f1-j1-getPrimeroCasesV2.js | 131 +++++++++++++++++------------ 1 file changed, 76 insertions(+), 55 deletions(-) diff --git a/jobs/V2/f1-j1-getPrimeroCasesV2.js b/jobs/V2/f1-j1-getPrimeroCasesV2.js index f82848f..9d4bd80 100644 --- a/jobs/V2/f1-j1-getPrimeroCasesV2.js +++ b/jobs/V2/f1-j1-getPrimeroCasesV2.js @@ -1,7 +1,7 @@ // Either use a manual cursor, or take the cursor from the last run. fn(state => { console.log('Last sync end date:', state.lastRunDateTime || 'undefined; using manual cursor...'); - const manualCursor = '2024-04-09T00:00:00.000Z'; //for missing referral: '2023-08-13T06:00:00.862Z'; + const manualCursor = '2024-04-09T00:00:00.000Z'; //for missing referral: '2023-08-13T06:00:00.862Z'; const currentAttempt = new Date().toISOString(); console.log('Current attempt time:', currentAttempt); @@ -16,64 +16,85 @@ fn(state => ({ ...state, data: {}, references: [], referralIds: [] })); // GET Primero cases with oscar referrals // User Story 1: Generating government referrals // #1 - Request oscar referrals only =========================================== -getCases( - { - remote: true, - last_updated_at: state => `${state.cursor}..`, - //last_updated_at: '2023-08-24T00:00:00.862Z..', //for troubleshooting - //last_updated_at: state => `${state.cursor}..2023-08-14T15:00:00.862Z`, //for troubleshooting - page: 1, - per: 10000, - // These cases have been recently updated and MIGHT have a new referral to send to Oscar; we check & filter below - // workflow: 'referral_to_oscar', //REMOVED July '23 bc we should rely on services, not case-level statuses - }, - { withReferrals: true }, - state => { - const oscarRefs = state.data; - const referralIds = oscarRefs - .map(c => - c.referrals - .map(x => { - console.log('referral:', 'id:', x.id, x.status, x.created_at, x.service_record_id); - return x; - }) - .filter(r => new Date(r.created_at) >= new Date(state.cursor)) //Only send referrals created after the last sync - .filter(r => r.status === 'in_progress') // And only send referrals where status 'in progress' - .map(r => r.service_record_id) - ) - .flat(); - - console.log('Detected referral Ids:', referralIds); - - // we filter the services_section in each oscarRef to return only the services that have/have not been sent to Oscar - const sentOscarRefs = oscarRefs - .filter(c => c.services_section) - .map(c => ({ - ...c, - services_section: c.services_section.filter(service => { - if (referralIds.includes(service.unique_id)) { - // console.log('Detected service to refer to oscar :', service.unique_id); - // console.log('Service response type :', service.service_response_type); - return true; - } - // console.log('N/A, service not recently referred to Oscar :', service.unique_id); +fn(async state => { + const perPage = 20; + let currentPage = 1; + let totalItems = 0; + let allCases = []; + + do { + const cases = await getCases( + { + remote: true, + last_updated_at: state => `${state.cursor}..`, + //last_updated_at: '2023-08-24T00:00:00.862Z..', //for troubleshooting + //last_updated_at: state => `${state.cursor}..2023-08-14T15:00:00.862Z`, //for troubleshooting + page: currentPage, + per: perPage, + // These cases have been recently updated and MIGHT have a new referral to send to Oscar; we check & filter below + // workflow: 'referral_to_oscar', //REMOVED July '23 bc we should rely on services, not case-level statuses + }, + { withReferrals: true } + )(state); + + const { + metadata: { total }, + data, + } = cases; + + totalItems = total; + allCases = allCases.concat(data); + currentPage++; + } while ((currentPage - 1) * perPage < totalItems); + + return { ...state, data: allCases }; +}); + +fn(state => { + const oscarRefs = state.data; + const referralIds = oscarRefs + .map(c => + c.referrals + .map(x => { + console.log('referral:', 'id:', x.id, x.status, x.created_at, x.service_record_id); + return x; + }) + .filter(r => new Date(r.created_at) >= new Date(state.cursor)) //Only send referrals created after the last sync + .filter(r => r.status === 'in_progress') // And only send referrals where status 'in progress' + .map(r => r.service_record_id) + ) + .flat(); + + console.log('Detected referral Ids:', referralIds); + + // we filter the services_section in each oscarRef to return only the services that have/have not been sent to Oscar + const sentOscarRefs = oscarRefs + .filter(c => c.services_section) + .map(c => ({ + ...c, + services_section: c.services_section.filter(service => { + if (referralIds.includes(service.unique_id)) { + // console.log('Detected service to refer to oscar :', service.unique_id); // console.log('Service response type :', service.service_response_type); - return false; - }), - })); + return true; + } + // console.log('N/A, service not recently referred to Oscar :', service.unique_id); + // console.log('Service response type :', service.service_response_type); + return false; + }), + })); - const pendingRefsToSend = sentOscarRefs.filter( - c => c.services_section && c.services_section.length > 0 - ); + const pendingRefsToSend = sentOscarRefs.filter( + c => c.services_section && c.services_section.length > 0 + ); - console.log( - 'Case IDs of pending referrals to send to Oscar ::', - pendingRefsToSend ? pendingRefsToSend.map(c => c.case_id) : null - ); + console.log( + 'Case IDs of pending referrals to send to Oscar ::', + pendingRefsToSend ? pendingRefsToSend.map(c => c.case_id) : null + ); - return { ...state, oscarRefs: pendingRefsToSend, referralIds, data: {}, references: [] }; - } -); + return { ...state, oscarRefs: pendingRefsToSend, referralIds, data: {}, references: [] }; +}); // NEW: Get decisions to send back to OSCAR ================================================// getCases( { From 0e883569ed64e34196572c74ff8045a75addfdd0 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Thu, 4 Jul 2024 09:49:34 +0300 Subject: [PATCH 2/4] increase perPage and add logging --- jobs/V2/f1-j1-getPrimeroCasesV2.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jobs/V2/f1-j1-getPrimeroCasesV2.js b/jobs/V2/f1-j1-getPrimeroCasesV2.js index 9d4bd80..e8e2bf8 100644 --- a/jobs/V2/f1-j1-getPrimeroCasesV2.js +++ b/jobs/V2/f1-j1-getPrimeroCasesV2.js @@ -17,12 +17,13 @@ fn(state => ({ ...state, data: {}, references: [], referralIds: [] })); // User Story 1: Generating government referrals // #1 - Request oscar referrals only =========================================== fn(async state => { - const perPage = 20; + const perPage = 100; let currentPage = 1; let totalItems = 0; let allCases = []; do { + console.log('Fetching cases and their referralls...'); const cases = await getCases( { remote: true, @@ -37,6 +38,7 @@ fn(async state => { { withReferrals: true } )(state); + console.log(`Fetched ${cases.length} cases`); const { metadata: { total }, data, From e5f4bab17f2f26c72e96d4f078faa2c71ef1d9bb Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Fri, 5 Jul 2024 10:46:50 +0300 Subject: [PATCH 3/4] fix typo --- jobs/V2/f1-j1-getPrimeroCasesV2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/V2/f1-j1-getPrimeroCasesV2.js b/jobs/V2/f1-j1-getPrimeroCasesV2.js index e8e2bf8..4b3e85a 100644 --- a/jobs/V2/f1-j1-getPrimeroCasesV2.js +++ b/jobs/V2/f1-j1-getPrimeroCasesV2.js @@ -23,7 +23,7 @@ fn(async state => { let allCases = []; do { - console.log('Fetching cases and their referralls...'); + console.log('Fetching cases and their referrals...'); const cases = await getCases( { remote: true, From 50ff261e08797d913e43d20756989f554b90a3a2 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Fri, 5 Jul 2024 11:16:08 +0300 Subject: [PATCH 4/4] update log --- jobs/V2/f1-j1-getPrimeroCasesV2.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/V2/f1-j1-getPrimeroCasesV2.js b/jobs/V2/f1-j1-getPrimeroCasesV2.js index 4b3e85a..10dfe3c 100644 --- a/jobs/V2/f1-j1-getPrimeroCasesV2.js +++ b/jobs/V2/f1-j1-getPrimeroCasesV2.js @@ -23,7 +23,7 @@ fn(async state => { let allCases = []; do { - console.log('Fetching cases and their referrals...'); + console.log(`Fetching page ${currentPage} of cases and their referrals...`); const cases = await getCases( { remote: true,