Record update thread: use one thread per survey (#2863)

* refactoring record update thread * records update thread (WIP) * record update thread: fixed node update * kill threads on survey delete and survey publish (WIP) * record service: fixed import of webapp dependency * fixed delete preview record on checkout * code cleanup * code cleanup * code cleanup * initialize record on check-in * use latest version of arena-server * user service: code cleanup --------- Co-authored-by: Stefano Ricci <[email protected]> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
openforis · Jun 15, 2023 · ba0652b · ba0652b
1 parent 3f44d54
commit ba0652b
Show file tree

Hide file tree

Showing 29 changed files with 605 additions and 444 deletions.
diff --git a/package.json b/package.json
@@ -105,8 +105,8 @@
     "@mui/material": "^5.12.3",
     "@mui/x-data-grid": "^5.17.20",
     "@mui/x-date-pickers": "^5.0.14",
-    "@openforis/arena-core": "^0.0.142",
-    "@openforis/arena-server": "^0.1.24",
+    "@openforis/arena-core": "^0.0.143",
+    "@openforis/arena-server": "^0.1.25",
     "@sendgrid/mail": "^7.7.0",
     "@shopify/draggable": "^1.0.0-beta.8",
     "ace-builds": "^1.19.0",

diff --git a/server/modules/auth/api/authApi.js b/server/modules/auth/api/authApi.js
@@ -51,7 +51,7 @@ export const init = (app) => {
     try {
       // Before logout checkOut record if there's an opened thread
       const socketId = Request.getSocketId(req)
-      RecordService.dissocSocketFromRecordThread(socketId)
+      RecordService.dissocSocketFromUpdateThread(socketId)
 
       req.logout((err) => {
         if (err) {

diff --git a/server/modules/record/api/recordApi.js b/server/modules/record/api/recordApi.js
@@ -28,14 +28,14 @@ export const init = (app) => {
     try {
       const user = Request.getUser(req)
       const { surveyId } = Request.getParams(req)
-      const record = Request.getBody(req)
+      const recordToCreate = Request.getBody(req)
       const socketId = Request.getSocketId(req)
 
-      if (Record.getOwnerUuid(record) !== User.getUuid(user)) {
+      if (Record.getOwnerUuid(recordToCreate) !== User.getUuid(user)) {
         throw new Error('Error record create. User is different')
       }
 
-      await RecordService.createRecord(socketId, user, surveyId, record)
+      await RecordService.createRecord({ socketId, user, surveyId, recordToCreate })
 
       sendOk(res)
     } catch (error) {
@@ -59,12 +59,12 @@ export const init = (app) => {
   app.post('/survey/:surveyId/record/:recordUuid/node', requireRecordEditPermission, async (req, res, next) => {
     try {
       const user = Request.getUser(req)
-      const { surveyId } = Request.getParams(req)
+      const { surveyId, cycle, draft } = Request.getParams(req)
       const node = Request.getJsonParam(req, 'node')
       const file = Request.getFile(req)
       const socketId = Request.getSocketId(req)
 
-      await RecordService.persistNode(socketId, user, surveyId, node, file)
+      await RecordService.persistNode({ socketId, user, surveyId, cycle, draft, node, file })
 
       sendOk(res)
     } catch (error) {
@@ -329,7 +329,7 @@ export const init = (app) => {
       const user = Request.getUser(req)
       const socketId = Request.getSocketId(req)
 
-      const record = await RecordService.checkIn(socketId, user, surveyId, recordUuid, draft)
+      const record = await RecordService.checkIn({ socketId, user, surveyId, recordUuid, draft })
 
       res.json({ record })
     } catch (error) {
@@ -394,11 +394,11 @@ export const init = (app) => {
   })
 
   app.delete('/survey/:surveyId/record/:recordUuid/node/:nodeUuid', requireRecordEditPermission, (req, res) => {
-    const { surveyId, recordUuid, nodeUuid } = Request.getParams(req)
+    const { surveyId, cycle, draft, recordUuid, nodeUuid } = Request.getParams(req)
     const user = Request.getUser(req)
     const socketId = Request.getSocketId(req)
 
-    RecordService.deleteNode(socketId, user, surveyId, recordUuid, nodeUuid)
+    RecordService.deleteNode({ socketId, user, surveyId, cycle, draft, recordUuid, nodeUuid })
     sendOk(res)
   })
 }
diff --git a/server/modules/record/manager/recordManager.js b/server/modules/record/manager/recordManager.js
@@ -29,22 +29,51 @@ export const { persistNodesToRDB } = NodeRdbManager
 // ==== READ
 
 export const fetchRecordsSummaryBySurveyId = async (
-  { surveyId, cycle, offset, limit, sortBy, sortOrder, search, step = null, recordUuid = null },
+  {
+    surveyId,
+    cycle,
+    offset,
+    limit,
+    sortBy,
+    sortOrder,
+    search,
+    step = null,
+    recordUuid = null,
+    includeRootKeyValues = true,
+    includePreview = false,
+  },
   client = db
 ) => {
   const surveyInfo = await SurveyRepository.fetchSurveyById({ surveyId, draft: true }, client)
   const nodeDefsDraft = Survey.isFromCollect(surveyInfo) && !Survey.isPublished(surveyInfo)
 
-  const nodeDefRoot = await NodeDefRepository.fetchRootNodeDef(surveyId, nodeDefsDraft, client)
-  const nodeDefKeys = await NodeDefRepository.fetchRootNodeDefKeysBySurveyId(
-    surveyId,
-    NodeDef.getUuid(nodeDefRoot),
-    nodeDefsDraft,
-    client
-  )
+  const nodeDefRoot = includeRootKeyValues
+    ? await NodeDefRepository.fetchRootNodeDef(surveyId, nodeDefsDraft, client)
+    : null
+  const nodeDefKeys = includeRootKeyValues
+    ? await NodeDefRepository.fetchRootNodeDefKeysBySurveyId(
+        surveyId,
+        NodeDef.getUuid(nodeDefRoot),
+        nodeDefsDraft,
+        client
+      )
+    : null
 
   const list = await RecordRepository.fetchRecordsSummaryBySurveyId(
-    { surveyId, cycle, nodeDefRoot, nodeDefKeys, offset, limit, sortBy, sortOrder, search, step, recordUuid },
+    {
+      surveyId,
+      cycle,
+      nodeDefRoot,
+      nodeDefKeys,
+      offset,
+      limit,
+      sortBy,
+      sortOrder,
+      search,
+      step,
+      recordUuid,
+      includePreview,
+    },
     client
   )
 
@@ -54,8 +83,14 @@ export const fetchRecordsSummaryBySurveyId = async (
   }
 }
 
-export const fetchRecordSummary = async ({ surveyId, recordUuid }, client = db) => {
-  const { list } = await fetchRecordsSummaryBySurveyId({ surveyId, recordUuid }, client)
+export const fetchRecordSummary = async (
+  { surveyId, recordUuid, includeRootKeyValues = true, includePreview = false },
+  client = db
+) => {
+  const { list } = await fetchRecordsSummaryBySurveyId(
+    { surveyId, recordUuid, includeRootKeyValues, includePreview },
+    client
+  )
   return list[0]
 }
 

diff --git a/server/modules/record/repository/recordRepository.js b/server/modules/record/repository/recordRepository.js
@@ -130,8 +130,8 @@ export const countRecordsBySurveyIdGroupedByStep = async ({ surveyId, cycle }, c
 export const fetchRecordsSummaryBySurveyId = async (
   {
     surveyId,
-    nodeDefRoot,
-    nodeDefKeys,
+    nodeDefRoot = null,
+    nodeDefKeys = null,
     cycle = null,
     step = null,
     offset = 0,
@@ -140,25 +140,26 @@ export const fetchRecordsSummaryBySurveyId = async (
     sortOrder = 'DESC',
     search = false,
     recordUuid = null,
+    includePreview = false,
   },
   client = db
 ) => {
   const rootEntityTableAlias = 'n0'
   const getNodeDefKeyColumnName = NodeDefTable.getColumnName
   const getNodeDefKeyColAlias = NodeDef.getName
-  const nodeDefKeysColumnNamesByAlias = nodeDefKeys.reduce(
+  const nodeDefKeysColumnNamesByAlias = nodeDefKeys?.reduce(
     (acc, key) => ({ ...acc, [getNodeDefKeyColAlias(key)]: getNodeDefKeyColumnName(key) }),
     {}
   )
   const nodeDefKeysSelect = nodeDefKeys
-    .map(
+    ?.map(
       (nodeDefKey) =>
         `${rootEntityTableAlias}.${getNodeDefKeyColumnName(nodeDefKey)} as "${getNodeDefKeyColAlias(nodeDefKey)}"`
     )
     .join(', ')
 
   const nodeDefKeysSelectSearch = nodeDefKeys
-    .map(
+    ?.map(
       (nodeDefKey) =>
         ` (${rootEntityTableAlias}.${getNodeDefKeyColumnName(nodeDefKey)})::text ilike '%$/search:value/%'`
     )
@@ -172,11 +173,19 @@ export const fetchRecordsSummaryBySurveyId = async (
     FROM ${schema}.node
     GROUP BY record_uuid
   `
+  const recordsSelectWhereConditions = []
+  if (!includePreview) recordsSelectWhereConditions.push('r.preview = FALSE')
+  if (!A.isNull(cycle)) recordsSelectWhereConditions.push('r.cycle = $/cycle/')
+  if (!A.isNull(step)) recordsSelectWhereConditions.push('r.step = $/step/')
+  if (!A.isNull(recordUuid)) recordsSelectWhereConditions.push('r.uuid = $/recordUuid/')
+
   const recordsSelect = `
     SELECT 
         r.uuid, 
         r.owner_uuid, 
+        r.cycle,
         r.step, 
+        r.preview, 
         ${DbUtils.selectDate('r.date_created', 'date_created')}, 
         r.validation,
         node_last_modified.date_modified
@@ -185,11 +194,7 @@ export const fetchRecordsSummaryBySurveyId = async (
       LEFT OUTER JOIN 
         node_last_modified
         ON r.uuid = node_last_modified.record_uuid
-    WHERE 
-      r.preview = FALSE 
-      ${A.isNull(cycle) ? '' : 'AND r.cycle = $/cycle/'}
-      ${A.isNull(step) ? '' : 'AND r.step = $/step/'}
-      ${A.isNull(recordUuid) ? '' : 'AND r.uuid = $/recordUuid/'}
+    ${recordsSelectWhereConditions.length > 0 ? `WHERE ${recordsSelectWhereConditions.join(' AND ')}` : ''}
     ORDER BY r.date_created DESC
   `
 
@@ -214,15 +219,19 @@ export const fetchRecordsSummaryBySurveyId = async (
     -- GET OWNER NAME
     JOIN "user" u
       ON r.owner_uuid = u.uuid
-    -- join with root entity table to get node key values 
-    LEFT OUTER JOIN
-      ${SchemaRdb.getName(surveyId)}.${NodeDefTable.getViewName(nodeDefRoot)} as ${rootEntityTableAlias}
-    ON r.uuid = ${rootEntityTableAlias}.record_uuid
+      ${
+        nodeDefRoot && nodeDefKeys?.length > 0
+          ? `-- join with root entity table to get node key values 
+      LEFT OUTER JOIN
+        ${SchemaRdb.getName(surveyId)}.${NodeDefTable.getViewName(nodeDefRoot)} as ${rootEntityTableAlias}
+      ON r.uuid = ${rootEntityTableAlias}.record_uuid`
+          : ''
+      }
 
     ${whereCondition}
 
     ORDER BY ${
-      Object.keys(nodeDefKeysColumnNamesByAlias).includes(toSnakeCase(sortBy))
+      nodeDefKeysColumnNamesByAlias && Object.keys(nodeDefKeysColumnNamesByAlias).includes(toSnakeCase(sortBy))
         ? `${rootEntityTableAlias}.${nodeDefKeysColumnNamesByAlias[toSnakeCase(sortBy)]}`
         : `r.${toSnakeCase(sortBy)}`
     } ${sortOrder}