Merge pull request #226 from OpenFn/form-sharing-updates

Form sharing updates

bns/getArchivedFormsListFromSheets.js

@@ -0,0 +1,69 @@
+//== Job to be used for getting a list of "archived" Kobo forms from sheets to auto-sync  ==//
+// This can be run on-demand at any time by clicking "run" //
+getValues(
+  '1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY',
+  'wcs-bns-ARCHIVED!A:O', //get Deployed forms list from Sheet
+  state => {
+    const [headers, ...values] = state.data.values;
+
+    const mapHeaderToValue = value => {
+      return headers.reduce((obj, header) => {
+        obj[header] = value[headers.indexOf(header)];
+        return obj;
+      }, {});
+    };
+
+    state.sheetsData = values
+      .filter(
+        item => item.includes('TRUE') //return forms where auto-sync = TRUE
+        //&& item.includes('bns_survey', 'nrgt_current')
+      )
+      .map(item => mapHeaderToValue(item));
+
+    return state;
+  }
+);
+
+fn(state => {
+  const { sheetsData } = state;
+
+  // Set a manual cursor if you'd like to only fetch data after this date...
+  //e.g., '2023-01-01T23:51:45.491+01:00'
+  //   const manualCursor = ''; //lastUsed: 2024-04-01T00:00:00.000Z
+  //   console.log('manualCursor defined?', manualCursor);
+  //...otherwise the job will use this dynamicCursor
+  // const dynamicCursor = getTodayISODate();
+
+  //   function getTodayISODate() {
+  //     const today = new Date();
+  //     today.setUTCHours(0, 0, 0, 0); // Set hours, minutes, seconds, and milliseconds to 0
+  //     return today.toISOString(); // Convert to ISO string
+  //   }
+
+  //   const cursorValue = manualCursor || dynamicCursor;
+  //   console.log('Cursor value to use in query:', cursorValue);
+
+  const formsList = sheetsData.map(survey => ({
+    formId: survey.uid,
+    tag: survey.tag,
+    name: survey.name,
+  }));
+
+  console.log('# of archived forms detected in Sheet:: ', formsList.length);
+  console.log(
+    'List of forms to re-sync:: ',
+    JSON.stringify(formsList, null, 2)
+  );
+
+  state.data = {
+    surveys: sheetsData.map(survey => ({
+      formId: survey.uid,
+      tag: survey.tag,
+      name: survey.name,
+      owner: survey.owner,
+      url: `https://kf.kobotoolbox.org/api/v2/assets/${survey.uid}/data/?format=json`,
+      //query: `&query={"end":{"$gte":"${cursorValue}"}}`, //get ALL forms for historical job
+    })),
+  };
+  return state;
+});

bns/getKoboDataHistorical.js

@@ -0,0 +1,29 @@
+// Here we fetch submissions for all "Archived" forms in GoogleSheet
+// NOTE: See linked job "[BNS-1B] 1.Get FormsList (Historical)" for GoogleSheet query logic
+//**********************************************************//
+each(dataPath('surveys[*]'), state => {
+  const { url, query, tag, formId, name, owner } = state.data;
+  return get(`${url}${query}`, {}, state => {
+    state.data.submissions = state.data.results.map((submission, i) => {
+      return {
+        i,
+        // Here we append the tags defined above to the Kobo form submission data
+        form: tag,
+        formName: name,
+        formOwner: owner,
+        body: submission,
+      };
+    });
+    const count = state.data.submissions.length;
+    console.log('Finding historical forms to resync...');
+    console.log(`Fetched ${count} submissions from ${formId} (${tag}).`);
+    //Once we fetch the data, we want to post each individual Kobo survey
+    //back to the OpenFn inbox to run through the jobs =========================
+    return each(dataPath('submissions[*]'), state => {
+      console.log(`Posting ${state.data.i + 1} of ${count}...`);
+      return post(state.configuration.openfnInboxUrl, {
+        body: state => state.data,
+      })(state);
+    })(state);
+  })(state);
+});

docs/6-form-sharing.md

@@ -7,7 +7,7 @@ permalink: /form-sharing/
 
 # Kobo Form Sharing Notification for WCS Admins
 
-Lists of "deployed" and "archived" Kobo forms for data collection are stored in [this Google Sheet](https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit?gid=1559623602#gid=1559623602). 
+Lists of "deployed" and "archived" Kobo BNS/NRGT forms for data collection are stored in [this Google Sheet](https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit?gid=1559623602#gid=1559623602). 
 
 The below "Form Sharing" workflow has been configured in OpenFn. See the [form-sharing](https://github.com/OpenFn/ConSoSci/tree/master/form-sharing) directory for the underlying code. 
 
@@ -20,6 +20,16 @@ When the workflow runs, it will:
 
 ![form-sharing](./form-sharing.png)
 
+## Managing the Forms Lists in Google Sheets
+[This Sheet](https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit?gid=1559623602#gid=1559623602) is now the source of truth for what Kobo forms should or should not be synced to the ConSoSci database. Update the column `automate_sync` to true/false to specify whether these forms should be included when either the [BNS-1A Ongoing]() or [BNS-1B Historical] "Get forms" jobs run.
+
 ## Specifications
 - Original [Github technical specification](https://github.com/OpenFn/ConSoSci/issues/206) and the [workflow diagram (v2)](https://lucid.app/lucidchart/346b8e5c-6fb6-4a33-9d02-53e5059bd698/edit?invitationId=inv_d1431bce-05ae-4005-9b6a-9c279141a3a3&page=0_0#)
 - [Change request](https://github.com/OpenFn/ConSoSci/issues/224) for how archived forms are managed 
+
+## How to re-process Kobo form submissions & re-sync historical data
+You can either: 
+1. add the form to the [archived forms list](https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit?gid=1965562058#gid=1965562058), set the column L `automate-sync: true` and run the historical job to re-sync _all_ submissions for that form, or 
+2. specify a manual cursor in the [Get FormsList Ongoing job](https://v1.openfn.org/projects/consosci/jobs/E7cauG). 
+
+See this video walkthrough: 

form-sharing/4-upsertAsanaTask.js

@@ -12,7 +12,7 @@ fn(state => {
       assignee_section: '1207247884457665', //OLD General Section: '1203181218738601',
       assignee: '473999120764595',
       due_on: dueDate,
-      notes: `New form added to OpenFn: ${form.name}. Please review the Google Sheet to (1) update columns L, N, & O, and (2) update column E (look for cells where it says "ADD MANUALLY" to add any values missing e.g., "Instance"): https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,
+      notes: `New form added to OpenFn: ${form.name} (uid: ${form.uid}). Please review the Google Sheet to (1) update columns L, N, & O, and (2) update column E (look for cells where it says "ADD MANUALLY" to add any values missing e.g., "Instance"): https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,
     };
   });
 
@@ -23,14 +23,20 @@ fn(state => {
       assignee_section: '1207247884457665', //OLD General Section: '1203181218738601',
       assignee: '473999120764595',
       due_on: dueDate,
-      notes: `Kobo form was archived: ${form.name}. Please review the Google Sheet to (1) confirm this is correct, (2) remove from the "Deployed" sheet if you want to remove from the OpenFn Sync, and (3) update notes in the "Archived" sheet: https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,
+      notes: `Kobo form was archived: ${form.name} (uid: ${form.uid}). Please review the Google Sheet to (1) confirm this is correct, (2) remove from the "Deployed" sheet if you want to remove from the OpenFn Sync, and (3) update notes in the "Archived" sheet: https://docs.google.com/spreadsheets/d/1s7K3kxzm5AlpwiALattyc7D9_aIyqWmo2ubcQIUlqlY/edit#gid=1559623602`,
     };
   });
 
   console.log('# of New Form Asana Tasks to add:: ', state.asanaTasks.length);
   console.log('New form alert tasks to upsert:: ', state.asanaTasks);
-  console.log('# of Archibed Form Asana Tasks to add:: ', state.archivedFormsTasks.length);
-  console.log('Archived form alert tasks to upsert:: ', state.archivedFormsTasks);
+  console.log(
+    '# of Archibed Form Asana Tasks to add:: ',
+    state.archivedFormsTasks.length
+  );
+  console.log(
+    'Archived form alert tasks to upsert:: ',
+    state.archivedFormsTasks
+  );
   return state;
 });