fix(fetch): Allow to set the fetch batch size limit with a cli argument

postalsys · Mar 6, 2024 · f5daf91 · f5daf91
1 parent de45851
commit f5daf91
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/lib/consts.js b/lib/consts.js
@@ -167,7 +167,7 @@ module.exports = {
     FETCH_RETRY_INTERVAL: 1000,
     FETCH_RETRY_EXPONENTIAL: 2,
     FETCH_RETRY_MAX: 60 * 1000,
-    MAX_FETCH_RANGE: Number((process.env.EENGINE_MAX_FETCH_RANGE || '').toString().trim()) || 1000,
+    DEFAULT_FETCH_BATCH_SIZE: 1000,
 
     generateWebhookTable() {
         let entries = [];

diff --git a/lib/mailbox.js b/lib/mailbox.js
@@ -9,12 +9,14 @@ const {
     download,
     filterEmptyObjectValues,
     validUidValidity,
-    calculateFetchBackoff
+    calculateFetchBackoff,
+    readEnvValue
 } = require('./tools');
 const msgpack = require('msgpack5')();
 const he = require('he');
 const libmime = require('libmime');
 const settings = require('./settings');
+const config = require('wild-config');
 const { bounceDetect } = require('./bounce-detect');
 const { arfDetect } = require('./arf-detect');
 const appendList = require('./append-list');
@@ -38,9 +40,11 @@ const {
     REDIS_PREFIX,
     MAX_INLINE_ATTACHMENT_SIZE,
     MAX_ALLOWED_DOWNLOAD_SIZE,
-    MAX_FETCH_RANGE
+    DEFAULT_FETCH_BATCH_SIZE
 } = require('./consts');
 
+const FETCH_BATCH_SIZE = Number(readEnvValue('EENGINE_FETCH_BATCH_SIZE') || config.service.fetchBatchSize) || DEFAULT_FETCH_BATCH_SIZE;
+
 // Do not check for flag updates using full sync more often than this value
 const FULL_SYNC_DELAY = 30 * 60 * 1000;
 
@@ -54,7 +58,7 @@ function getFetchRange(totalMessages, lastRange) {
     if (startUid > totalMessages) {
         return false;
     }
-    let endMarker = startUid + MAX_FETCH_RANGE - 1;
+    let endMarker = startUid + FETCH_BATCH_SIZE - 1;
     if (endMarker >= totalMessages) {
         endMarker = '*';
     }