Files storage: allow specifying max files (#3585)

* Map: renamed altitude into elevation

* survey files: get max space from DB

* prepare editing

* added item edit button bar

* max files size editor

* files size formatting: use power of 1024 by default

* updated arena-server version

* code cleanup

---------

Co-authored-by: Stefano Ricci <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

core/auth/authorizer.js

@@ -53,6 +53,7 @@ export const canViewTemplates = (user) => User.isSystemAdmin(user)
 
 // UPDATE
 export const canEditSurvey = _hasSurveyPermission(permissions.surveyEdit)
+export const canEditSurveyConfig = (user) => User.isSystemAdmin(user)
 export const canEditTemplates = (user) => User.isSystemAdmin(user)
 export const canRefreshAllSurveyRdbs = (user) => User.isSystemAdmin(user)
 

core/i18n/resources/en/common.js

@@ -437,6 +437,10 @@ Thank you and enjoy **$t(common.appNameFull)**!`,
     },
     surveyDeleted: 'Survey {{surveyName}} has been deleted',
     surveyInfo: {
+      configuration: {
+        title: 'Configuration',
+        filesTotalSpace: 'Files total space (GB)',
+      },
       confirmDeleteCycleHeader: 'Delete this cycle?',
       confirmDeleteCycle: `Are you sure you want to delete the cycle {{cycle}}?\n\n$t(common.cantUndoWarning)\n\n
 If there are records associated to this cycle, they will be deleted.`,

core/numberConversionUtils.js

@@ -0,0 +1,75 @@
+import { Objects } from '@openforis/arena-core'
+
+const areaUnits = {
+  squareMeter: 'squareMeter',
+  squareFoot: 'squareFoot',
+  acre: 'acre',
+  hectare: 'hectare',
+}
+
+const areaUnitToSquareMetersConversionFactor = {
+  [areaUnits.acre]: 4046.85642199999983859016,
+  [areaUnits.hectare]: 10000,
+  [areaUnits.squareMeter]: 1,
+  [areaUnits.squareFoot]: 0.09290304,
+}
+
+const lengthUnits = {
+  meter: 'meter',
+  foot: 'foot',
+}
+
+const lengthUnitToMetersConversionFactor = {
+  [lengthUnits.meter]: 1,
+  [lengthUnits.foot]: 0.3048,
+}
+
+const abbreviationByUnit = {
+  [areaUnits.squareMeter]: 'm²',
+  [areaUnits.squareFoot]: 'ft²',
+  [areaUnits.acre]: 'ac',
+  [areaUnits.hectare]: 'ha',
+  [lengthUnits.meter]: 'm',
+  [lengthUnits.foot]: 'ft',
+}
+
+const dataStorageUnits = {
+  byte: 'byte',
+  MB: 'MB',
+  GB: 'GB',
+}
+
+const dataStorageUnitToBytesConversionFactor = {
+  [dataStorageUnits.byte]: 1,
+  [dataStorageUnits.MB]: Math.pow(1024, 2),
+  [dataStorageUnits.GB]: Math.pow(1024, 3),
+}
+
+const squareMetersToUnit = (unit) => (value) =>
+  Objects.isNil(value) ? NaN : Number(value) / areaUnitToSquareMetersConversionFactor[unit]
+
+const metersToUnit = (unit) => (value) =>
+  Objects.isNil(value) ? NaN : Number(value) / lengthUnitToMetersConversionFactor[unit]
+
+const dataStorageBytesToUnit = (unit) => (bytes) =>
+  Objects.isNil(bytes) ? NaN : Number(bytes) / dataStorageUnitToBytesConversionFactor[unit]
+
+const dataStorageValueToBytes = (unit) => (value) =>
+  Objects.isNil(value) ? NaN : Number(value) * dataStorageUnitToBytesConversionFactor[unit]
+
+const dataStorageValueToUnit = (unitFrom, unitTo) => (value) => {
+  const bytes = dataStorageValueToBytes(unitFrom)(value)
+  return dataStorageBytesToUnit(unitTo)(bytes)
+}
+
+export const NumberConversionUtils = {
+  areaUnits,
+  lengthUnits,
+  abbreviationByUnit,
+  metersToUnit,
+  squareMetersToUnit,
+  dataStorageUnits,
+  dataStorageBytesToUnit,
+  dataStorageValueToBytes,
+  dataStorageValueToUnit,
+}

core/numberUtils.js

@@ -12,39 +12,6 @@ BigNumber.config({
   },
 })
 
-export const areaUnits = {
-  squareMeter: 'squareMeter',
-  squareFoot: 'squareFoot',
-  acre: 'acre',
-  hectare: 'hectare',
-}
-
-const areaUnitToSquareMetersConversionFactor = {
-  [areaUnits.acre]: 4046.85642199999983859016,
-  [areaUnits.hectare]: 10000,
-  [areaUnits.squareMeter]: 1,
-  [areaUnits.squareFoot]: 0.09290304,
-}
-
-export const lengthUnits = {
-  meter: 'meter',
-  foot: 'foot',
-}
-
-const lengthUnitToMetersConversionFactor = {
-  [lengthUnits.meter]: 1,
-  [lengthUnits.foot]: 0.3048,
-}
-
-export const abbreviationByUnit = {
-  [areaUnits.squareMeter]: 'm²',
-  [areaUnits.squareFoot]: 'ft²',
-  [areaUnits.acre]: 'ac',
-  [areaUnits.hectare]: 'ha',
-  [lengthUnits.meter]: 'm',
-  [lengthUnits.foot]: 'ft',
-}
-
 export const toNumber = (num) => (Objects.isEmpty(num) ? NaN : Number(num))
 
 export const isInteger = A.pipe(toNumber, Number.isInteger)
@@ -92,8 +59,11 @@ export const formatInteger = (value) => formatDecimal(value, 0)
  */
 export const mod = (modulus) => (value) => ((value % modulus) + modulus) % modulus
 
-export const squareMetersToUnit = (unit) => (value) =>
-  Objects.isNil(value) ? NaN : value / areaUnitToSquareMetersConversionFactor[unit]
-
-export const metersToUnit = (unit) => (value) =>
-  Objects.isNil(value) ? NaN : value / lengthUnitToMetersConversionFactor[unit]
+export const limit =
+  ({ minValue = null, maxValue = null }) =>
+  (value) => {
+    let result = Number(value)
+    if (minValue) result = Math.max(minValue, result)
+    if (maxValue) result = Math.min(maxValue, result)
+    return result
+  }

core/survey/_survey/surveyConfig.js

@@ -0,0 +1,3 @@
+export const configKeys = {
+  filesTotalSpace: 'filesTotalSpace', // total space to store files (in MB)
+}

core/survey/survey.js

@@ -8,6 +8,7 @@ import * as NodeDef from './nodeDef'
 
 import * as SurveySortKeys from './_survey/surveySortKeys'
 
+import * as SurveyConfig from './_survey/surveyConfig'
 import * as SurveyInfo from './_survey/surveyInfo'
 import * as SurveyCycle from './surveyCycle'
 import * as SurveyNodeDefs from './_survey/surveyNodeDefs'
@@ -60,6 +61,7 @@ export const newSurvey = ({
   [SurveyInfo.keys.template]: template,
 })
 
+export const { configKeys } = SurveyConfig
 export const { keys: infoKeys, status } = SurveyInfo
 export const { dependencyTypes } = SurveyDependencies
 export const { collectReportKeys, cycleOneKey, samplingPointDataCategoryName } = SurveyInfo

package.json

@@ -109,7 +109,7 @@
     "@mui/x-date-pickers": "^7.14.0",
     "@mui/x-tree-view": "^7.7.1",
     "@openforis/arena-core": "^0.0.204",
-    "@openforis/arena-server": "^0.1.34",
+    "@openforis/arena-server": "^0.1.35",
     "@reduxjs/toolkit": "^2.2.5",
     "@sendgrid/mail": "^8.1.3",
     "@shopify/draggable": "^1.1.3",

server/modules/auth/authApiMiddleware.js

@@ -101,6 +101,7 @@ export const requireSurveyCreatePermission = async (req, _res, next) => {
 }
 export const requireSurveyViewPermission = requireSurveyPermission(Authorizer.canViewSurvey)
 export const requireSurveyEditPermission = requireSurveyPermission(Authorizer.canEditSurvey)
+export const requireSurveyConfigEditPermission = requireSurveyPermission(Authorizer.canEditSurveyConfig)
 export const requireRecordCleansePermission = requireSurveyPermission(Authorizer.canCleanseRecords)
 export const requireSurveyRdbRefreshPermission = requirePermission(Authorizer.canRefreshAllSurveyRdbs)
 export const requireCanExportSurveysList = requirePermission(Authorizer.canExportSurveysList)

server/modules/record/manager/fileManager.js

@@ -5,12 +5,18 @@ import * as RecordFile from '@core/record/recordFile'
 import { db } from '@server/db/db'
 import * as Log from '@server/log/log'
 
+import * as SurveyRepository from '@server/modules/survey/repository/surveyRepository'
+
 import * as FileRepository from '../repository/fileRepository'
 import * as FileRepositoryFileSystem from '../repository/fileRepositoryFileSystem'
 import * as FileRepositoryS3Bucket from '../repository/fileRepositoryS3Bucket'
+import { NumberConversionUtils } from '@core/numberConversionUtils'
 
 const logger = Log.getLogger('FileManager')
 
+export const defaultSurveyFilesTotalSpaceMB = 10 * 1024 // in MB (=10 GB)
+export const maxSurveyFilesTotalSpaceMB = 100 * 1024 // in MB (=100 GB)
+
 export const fileContentStorageTypes = {
   db: 'db',
   fileSystem: 'fileSystem',
@@ -74,6 +80,12 @@ export const fetchFileContentAsBuffer = async ({ surveyId, fileUuid }, client =
   return streamToBuffer(contentStream)
 }
 
+export const fetchSurveyFilesTotalSpace = async ({ surveyId }) => {
+  const surveyTotalSpaceMB = await SurveyRepository.fetchFilesTotalSpace(surveyId)
+  const totalSpaceMB = surveyTotalSpaceMB ?? defaultSurveyFilesTotalSpaceMB
+  return NumberConversionUtils.dataStorageValueToBytes(NumberConversionUtils.dataStorageUnits.MB)(totalSpaceMB)
+}
+
 export const insertFile = async (surveyId, file, client = db) => {
   const storageType = getFileContentStorageType()
   const contentStoreFunction = contentStoreFunctionByStorageType[storageType]

server/modules/record/service/fileService.js

@@ -1,7 +1,8 @@
 import { Promises } from '@openforis/arena-core'
 
 import * as Log from '@server/log/log'
-import * as SurveyManager from '@server/modules/survey/manager/surveyManager'
+import * as SurveyRepository from '@server/modules/survey/repository/surveyRepository'
+
 import * as FileManager from '../manager/fileManager'
 
 const logger = Log.getLogger('FileService')
@@ -19,7 +20,7 @@ export const checkFilesStorage = async () => {
     return
   }
   logger.debug(`Moving survey files to new storage (if necessary)`)
-  const surveyIds = await SurveyManager.fetchAllSurveyIds()
+  const surveyIds = await SurveyRepository.fetchAllSurveyIds()
   let allSurveysFilesMoved = false
   let errorsFound = false
   await Promises.each(surveyIds, async (surveyId) => {
@@ -40,13 +41,8 @@ export const checkFilesStorage = async () => {
   }
 }
 
-// eslint-disable-next-line no-unused-vars
-const getSurveyFilesTotalSpace = async ({ surveyId }) => {
-  return 10 * Math.pow(1024, 3) // TODO make it configurable, fixed to 10 GB per survey now
-}
-
 export const fetchFilesStatistics = async ({ surveyId }) => {
-  const totalSpace = await getSurveyFilesTotalSpace({ surveyId })
+  const totalSpace = await FileManager.fetchSurveyFilesTotalSpace({ surveyId })
   const { total: usedSpace } = await FileManager.fetchCountAndTotalFilesSize({ surveyId })
   const availableSpace = Math.max(0, totalSpace - usedSpace)
 
@@ -58,7 +54,7 @@ export const fetchFilesStatistics = async ({ surveyId }) => {
 }
 
 export const cleanupAllSurveysFilesProps = async () => {
-  const surveyIds = await SurveyManager.fetchAllSurveyIds()
+  const surveyIds = await SurveyRepository.fetchAllSurveyIds()
   let count = 0
   for await (const surveyId of surveyIds) {
     const cleanedFiles = await FileManager.cleanupSurveyFilesProps({ surveyId })

server/modules/survey/api/surveyApi.js

@@ -146,7 +146,8 @@ export const init = (app) => {
     let surveyUpdated = survey
     if (Authorizer.canEditSurvey(user, Survey.getSurveyInfo(survey))) {
       const surveyId = Survey.getId(survey)
-      surveyUpdated = Survey.assocFilesStatistics(await FileService.fetchFilesStatistics({ surveyId }))(survey)
+      const filesStatistics = await FileService.fetchFilesStatistics({ surveyId })
+      surveyUpdated = Survey.assocFilesStatistics(filesStatistics)(survey)
     }
     res.json({ survey: surveyUpdated })
   }
@@ -295,6 +296,18 @@ export const init = (app) => {
     }
   })
 
+  app.put('/survey/:surveyId/config', AuthMiddleware.requireSurveyConfigEditPermission, async (req, res, next) => {
+    try {
+      const user = Request.getUser(req)
+      const { key, value } = Request.getBody(req)
+      const { surveyId } = Request.getParams(req)
+      const survey = await SurveyService.updateSurveyConfigurationProp({ surveyId, key, value })
+      await _sendSurvey({ survey, user, res })
+    } catch (error) {
+      next(error)
+    }
+  })
+
   // ==== DELETE
 
   app.delete('/survey/:surveyId', AuthMiddleware.requireSurveyEditPermission, async (req, res, next) => {

server/modules/survey/manager/surveyManager.js

@@ -10,9 +10,10 @@ import * as SurveyValidator from '@core/survey/surveyValidator'
 import * as NodeDef from '@core/survey/nodeDef'
 import * as NodeDefLayout from '@core/survey/nodeDefLayout'
 import * as User from '@core/user/user'
+import * as NumberUtils from '@core/numberUtils'
 import * as ObjectUtils from '@core/objectUtils'
-import * as Validation from '@core/validation/validation'
 import * as PromiseUtils from '@core/promiseUtils'
+import * as Validation from '@core/validation/validation'
 import SystemError from '@core/systemError'
 
 import { db } from '@server/db/db'
@@ -196,6 +197,7 @@ export const {
   fetchSurveysByName,
   fetchSurveyIdsAndNames,
   fetchDependencies,
+  fetchFilesTotalSpace,
 } = SurveyRepository
 
 export const fetchSurveyById = async ({ surveyId, draft = false, validate = false, backup = false }, client = db) => {
@@ -417,6 +419,22 @@ export const publishSurveyProps = async (surveyId, langsDeleted, client = db) =>
 export const unpublishSurveyProps = async (surveyId, client = db) =>
   SurveyRepository.unpublishSurveyProps(surveyId, client)
 
+export const updateSurveyConfigurationProp = async ({ surveyId, key, value }, client = db) => {
+  if (key !== Survey.configKeys.filesTotalSpace) {
+    throw new Error(`Configuration key update not supported: ${key}`)
+  }
+  const valueLimited = NumberUtils.limit({
+    minValue: FileManager.defaultSurveyFilesTotalSpaceMB,
+    maxValue: FileManager.maxSurveyFilesTotalSpaceMB,
+  })(value)
+  if (valueLimited === FileManager.defaultSurveyFilesTotalSpaceMB) {
+    await SurveyRepository.clearSurveyConfiguration({ surveyId }, client)
+  } else {
+    await SurveyRepository.updateSurveyConfigurationProp({ surveyId, key, value: String(valueLimited) }, client)
+  }
+  return fetchSurveyById({ surveyId, draft: true, validate: true }, client)
+}
+
 export const { removeSurveyTemporaryFlag, updateSurveyDependencyGraphs } = SurveyRepository
 
 // ====== DELETE

server/modules/survey/repository/surveyRepository.js

@@ -265,6 +265,15 @@ export const fetchDependencies = async (surveyId, client = db) =>
     R.prop('dependencies')
   )
 
+export const fetchFilesTotalSpace = async (surveyId, client = db) =>
+  client.oneOrNone(
+    `SELECT config#>'{${Survey.configKeys.filesTotalSpace}}' AS value
+     FROM survey 
+     WHERE id = $1`,
+    [surveyId],
+    R.prop('value')
+  )
+
 export const fetchTemporarySurveyIds = async ({ olderThan24Hours = false } = {}, client = db) =>
   client.map(
     `SELECT id 
@@ -354,6 +363,17 @@ export const removeSurveyTemporaryFlag = async ({ surveyId }, client = db) =>
     [surveyId]
   )
 
+export const updateSurveyConfigurationProp = async ({ surveyId, key, value }, client = db) =>
+  client.none(
+    `UPDATE survey
+    SET config = jsonb_set(coalesce(config, '{}'), '{${key}}', $/value/)
+    WHERE id = $/surveyId/`,
+    { surveyId, value }
+  )
+
+export const clearSurveyConfiguration = async ({ surveyId }, client = db) =>
+  client.none(`UPDATE survey SET config = null WHERE id = $1`, [surveyId])
+
 // ============== DELETE
 export const deleteSurvey = async (id, client = db) => client.one('DELETE FROM survey WHERE id = $1 RETURNING id', [id])
 

server/modules/survey/service/surveyService.js

@@ -137,6 +137,7 @@ export const {
   // UPDATE
   updateSurveyDependencyGraphs,
   updateSurveyProps,
+  updateSurveyConfigurationProp,
   // DELETE
   deleteTemporarySurveys,
   // UTILS