SHARD-1960: add max response size

[PudgyPug]

PR Type

enhancement, bug_fix

---

Description

• Introduced maxResponseSize configuration for response size limits.

• Replaced fetch with customFetch for HTTP requests.

• Added customAxios for size-limited Axios requests.

• Created customHttpFunctions.ts for custom HTTP utilities.

---

Changes walkthrough 📝

Relevant files

Enhancement

7 files Config.ts Add `maxResponseSize` to configuration                                      +3/-1      Cycles.ts Replace `fetch` with `customFetch` for HTTP requests          +2/-1      Data.ts Use `customFetch` for HTTP requests                                            +3/-2      P2P.ts Switch to `customFetch` for HTTP requests                                +3/-4      State.ts Implement `customFetch` for network calls                                +2/-1      txDigester.ts Use `customAxios` for HTTP requests                                            +2/-2      customHttpFunctions.ts Create custom HTTP functions with size limits                        +35/-0

---

Need help?

Type /help how to ... in the comments thread for any questions about PR-Agent usage.

Check out the documentation for more information.

[github-actions]

PR Reviewer Guide 🔍

Here are some key observations to aid the review process:

⏱️ Estimated effort to review: 3 🔵🔵🔵⚪⚪

🏅 Score: 85

🧪 No relevant tests

🔒 No security concerns identified

⚡ Recommended focus areas for review Possible Issue The customFetch function uses node-fetch's size option to limit response size, but this may not work as expected if the server sends a Content-Length header that is smaller than the actual response size. Consider adding additional checks or handling for this scenario. export async function customFetch( input: RequestInfo, init?: RequestInit, maxBytes?: number ): Promise<Response> { const downloadLimit = maxBytes ?? config.maxResponseSize // Pass the size option to node-fetch; if the response body exceeds this, an error is thrown. return await fetch(input, {...init, size: downloadLimit}) Configuration Consistency Ensure that the maxResponseSize configuration is consistently used across all HTTP requests and that any exceptions or errors are properly handled when the limit is exceeded. maxResponseSize: number } let config: Config = { ARCHIVER_IP: '127.0.0.1', ARCHIVER_PORT: 4000, ARCHIVER_HASH_KEY: '', ARCHIVER_PUBLIC_KEY: '', ARCHIVER_SECRET_KEY: '', ARCHIVER_LOGS: 'archiver-logs', ARCHIVER_DB: 'archiver-db', ARCHIVER_DATA: { cycleDB: 'cycles.sqlite3', accountDB: 'accounts.sqlite3', transactionDB: 'transactions.sqlite3', receiptDB: 'receipts.sqlite3', originalTxDataDB: 'originalTxsData.sqlite3', processedTxDB: 'processedTransactions.sqlite3', txDigestDB: 'txDigest.sqlite3', checkpointStatusDB: 'checkpointStatus.sqlite3', }, DATASENDER_TIMEOUT: 1000 * 60 * 5, RATE_LIMIT: 100, // 100 req per second, N_NODE_REJECT_PERCENT: 5, // Percentage of old nodes to remove from nodelist N_NODELIST: 10, // number of active node list GET /nodelist should emit but if the total active nodelist is less than said value it will emit all the node list. N_RANDOM_NODELIST_BUCKETS: 100, RECEIPT_CONFIRMATIONS: 5, STATISTICS: { save: true, interval: 1, }, ARCHIVER_MODE: 'release', // 'debug'/'release' DevPublicKey: '', dataLogWrite: true, dataLogWriter: { dirName: 'data-logs', maxLogFiles: 10, maxReceiptEntries: 10000, // Should be >= max TPS experienced by the network. maxCycleEntries: 500, maxOriginalTxEntries: 10000, // Should be >= max TPS experienced by the network. }, experimentalSnapshot: true, failedBucketsDir: 'failed-buckets', VERBOSE: false, useSerialization: true, useSyncV2: true, sendActiveMessage: false, globalNetworkAccount: process.env.GLOBAL_ACCOUNT || '1000000000000000000000000000000000000000000000000000000000000001', //this address will change in the future maxValidatorsToServe: 10, // max number of validators to serve accounts data during restore mode limitToArchiversOnly: true, verifyReceiptData: true, verifyReceiptSignaturesSeparately: true, verifyAccountData: true, verifyAppReceiptData: true, REQUEST_LIMIT: { MAX_ACCOUNTS_PER_REQUEST: 1000, MAX_RECEIPTS_PER_REQUEST: 100, MAX_ORIGINAL_TXS_PER_REQUEST: 100, MAX_CYCLES_PER_REQUEST: 100, MAX_BETWEEN_CYCLES_PER_REQUEST: 100, }, checkpoint: { bucketConfig: { BucketMatureAge: 11 * 60, // 11 minutes cycleAge: 60, // 60 seconds GiveUpAge: 20 * 60, // 20 minutes lastFailedBucketDuration: 5 * 60 * 1000, // 5 minutes RadixDepth: 2, // 2 nibbles (1 hex char) allowCheckpointUpdates: false, allowCheckpointStorage: false }, batchSize: 100, updateInterval: 60 * 1000, // 1 minute in milliseconds in milliseconds syncInterval: 10000, // 10 seconds in milliseconds maxCyclesToSync: 100, // Maximum number of cycles to sync in one go syncOnStartup: false, // Sync missing checkpoints on startup }, cycleRecordsCache: { enabled: false, }, newPOQReceipt: false, storeReceiptBeforeStates: true, waitingTimeForMissingTxData: 2000, // in ms gossipToMoreArchivers: true, randomGossipArchiversCount: 2, subscribeToMoreConsensors: true, extraConsensorsToSubscribe: 1, saveOnlyGossipData: false, stopGossipTxData: false, usePOQo: true, requiredVotesPercentage: 2 / 3, requiredMajorityVotesPercentage: 2 / 3, maxCyclesShardDataToKeep: 10, configChangeMaxCyclesToKeep: 5, configChangeMaxChangesToKeep: 1000, receiptLoadTrakerInterval: 10 * 1000, receiptLoadTrakerLimit: 10, lastActivityCheckInterval: 15 * 1000, lastActivityCheckTimeout: 30 * 1000, txDigest: { cycleDiff: 10, syncDelay: 20, apiServerPort: 8084, txCronSchedule: '*/5 * * * *', }, workerProcessesDebugLog: false, restrictFirstNodeSelectionByPublicKey: false, firstNodePublicKey: '', disableOffloadReceipt: false, disableOffloadReceiptForGlobalModification: true, restoreNGTsFromSnapshot: false, tickets: { allowedTicketSigners: { '0x002D3a2BfE09E3E29b6d38d58CaaD16EEe4C9BC5': 5, '0x80aF8E195B56aCC3b4ec8e2C99EC38957258635a': 5, '0x7Efbb31431ac7C405E8eEba99531fF1254fCA3B6': 5, '0xCc74bf387F6C102b5a7F828796C57A6D2D19Cb00': 5, '0x4ed5C053BF2dA5F694b322EA93dce949F3276B85': 5, '0xd31aBC7497aD8bC9fe8555C9eDe45DFd7FB3Bf6F': 5, '0xe7e4cc292b424C6D50d16F1Bb5BAB2032c486980': 5, '0xD815DA50966c19261B34Ffa3bE50A30A67D97456': 5, '0xE856B2365641eba73Bc430AAC1E8F930dA513D9D': 5, '0x8282F755e784414697421D4b59232E5d194e2262': 5, '0x353Ad64Df4fAe5EffF717A1c41BE6dEBee543129': 5, '0x9Ce1C3c114538c625aA2488b97fEb3723fdBB07B': 5, '0x6A83e4e4eB0A2c8f562db6BB64b02a9A6237B314': 5, '0x92E375E0c76CaE76D9DfBab17EE7B3B4EE407715': 5, '0xBD79B430CA932e2D89bb77ACaE7367a07471c2eA': 5, '0xfF2b584A947182c55BBc039BEAB78BC201D3AdDe': 5, '0xCeA068d8DCB4B4020D30a9950C00cF8408611F67': 5, '0x52F8d3DaA7b5FF25ca2bF7417E059aFe0bD5fB0E': 5, '0xF82BDA6Ef512e4219C6DCEea896E50e8180a5bff': 5, '0xA04A1B214a2537139fE59488820D4dA06516933f': 5, '0x550817e7B91244BBeFE2AD621ccD555A16B00405': 5, '0x84C55a4bFfff1ADadb9C46e2B60979F519dAf874': 5, '0x4563303BCE96D3f8d9C7fB94b36dfFC9d831871d': 5, '0xdA058F9c7Ce86C1D21DD5DBDeBad5ab5c785520a': 5, '0x891DF765C855E9848A18Ed18984B9f57cb3a4d47': 5, '0x7Fb9b1C5E20bd250870F87659E46bED410221f17': 5, '0x1e5e12568b7103E8B22cd680A6fa6256DD66ED76': 5, '0xa58169308e7153B5Ce4ca5cA515cC4d0cBE7770B': 5, }, minSigRequired: 1, requiredSecurityLevel: 5, }, maxRecordsPerRequest: 200, multisigKeysSyncFromNetworkInternal: 600, minCycleConfirmationsToSave: -1, nerfNonFoundationCertScores: true, formingNetworkCycleThreshold: 30, maxResponseSize: 15 * 1024 * 1024, // 15MB