From 102b749e79dc493d2f6ed93cf5035c58dc2b34a8 Mon Sep 17 00:00:00 2001 From: Denis DelGrosso Date: Wed, 8 Nov 2023 17:19:16 +0000 Subject: [PATCH] test: add ability to test many files without transfer manager --- internal-tooling/performPerformanceTest.ts | 95 +++++++++++++++++++++- internal-tooling/performanceTest.ts | 11 +-- internal-tooling/performanceUtils.ts | 1 + 3 files changed, 101 insertions(+), 6 deletions(-) diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts index a23f547c8..2306446ec 100644 --- a/internal-tooling/performPerformanceTest.ts +++ b/internal-tooling/performPerformanceTest.ts @@ -30,7 +30,7 @@ import { PERFORMANCE_TEST_TYPES, TestResult, } from './performanceUtils.js'; -import {Bucket} from '../src/index.js'; +import {Bucket, File} from '../src/index.js'; import {getDirName} from '../src/util.js'; const TEST_NAME_STRING = 'nodejs-perf-metrics'; @@ -64,6 +64,9 @@ async function main() { case PERFORMANCE_TEST_TYPES.RANGE_READ: results = await performRangedReadTest(); break; + case PERFORMANCE_TEST_TYPES.MANY_SMALL: + results = await performManyFilesTest(); + break; default: break; } @@ -214,4 +217,94 @@ async function performWriteReadTest(): Promise { return results; } +/** + * Performs an iteration of the many files test. + * + * @returns {Promise} Promise that resolves to an array of test results for the iteration. + */ +async function performManyFilesTest(): Promise { + const results: TestResult[] = []; + const fileSizeRange = getLowHighFileSize(argv.object_size as string); + const fileMap = new Map(); + let totalSizeInBytes = 0; + + for (let i = 0; i < (argv.num_objects as number); i++) { + const fileName = generateRandomFileName(TEST_NAME_STRING); + const file = bucket.file(`${fileName}`); + totalSizeInBytes += generateRandomFile( + fileName, + fileSizeRange.low, + fileSizeRange.high, + getDirName() + ); + fileMap.set(fileName, file); + } + + let iterationResult: TestResult = { + op: 'WRITE', + objectSize: totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: 'OK', + chunkSize: totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + let start = performance.now(); + for (const curFileName of fileMap.keys()) { + await bucket.upload(`${getDirName()}/${curFileName}`, { + validation: checkType, + }); + } + let end = performance.now(); + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(iterationResult); + + iterationResult = { + op: 'READ[0]', + objectSize: totalSizeInBytes, + appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES, + crc32cEnabled: checkType === 'crc32c', + md5Enabled: checkType === 'md5', + api: 'JSON', + elapsedTimeUs: 0, + cpuTimeUs: -1, + status: 'OK', + chunkSize: totalSizeInBytes, + workers: argv.workers as number, + library: 'nodejs', + transferSize: totalSizeInBytes, + transferOffset: 0, + bucketName: bucket.name, + }; + + for (let i = 0; i < DEFAULT_NUMBER_OF_READS; i++) { + start = performance.now(); + for (const curToDownload of fileMap.values()) { + await curToDownload.download({ + validation: checkType, + destination: path.join(getDirName(), curToDownload.name), + }); + } + end = performance.now(); + } + + iterationResult.elapsedTimeUs = Math.round((end - start) * 1000); + results.push(iterationResult); + for (const curToDelete of fileMap.values()) { + await curToDelete.delete({ignoreNotFound: true}); + cleanupFile(curToDelete.name, getDirName()); + } + + return results; +} + main(); diff --git a/internal-tooling/performanceTest.ts b/internal-tooling/performanceTest.ts index c5d88df5a..21824024d 100644 --- a/internal-tooling/performanceTest.ts +++ b/internal-tooling/performanceTest.ts @@ -60,6 +60,7 @@ function main() { * When the worker passes back the results, they are appended to the results file. */ function createWorker() { + const dirName = getDirName().replace('/src', '/internal-tooling'); iterationsRemaining--; log( `Starting new iteration. Current iterations remaining: ${iterationsRemaining}`, @@ -68,9 +69,10 @@ function createWorker() { let testPath = ''; if ( argv.test_type === PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE || - argv.test_type === PERFORMANCE_TEST_TYPES.RANGE_READ + argv.test_type === PERFORMANCE_TEST_TYPES.RANGE_READ || + argv.test_type === PERFORMANCE_TEST_TYPES.MANY_SMALL ) { - testPath = `${getDirName()}/performPerformanceTest.js`; + testPath = `${dirName}/performPerformanceTest.js`; } else if ( argv.test_type === PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MANY_FILES || @@ -79,7 +81,7 @@ function createWorker() { argv.test_type === PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MANY_FILES ) { - testPath = `${getDirName()}/performTransferManagerTest.js`; + testPath = `${dirName}/performTransferManagerTest.js`; } else if ( argv.test_type === PERFORMANCE_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS || @@ -87,9 +89,8 @@ function createWorker() { argv.test_type === PERFORMANCE_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS ) { - testPath = `${getDirName()}/performApplicationPerformanceTest.js`; + testPath = `${dirName}/performApplicationPerformanceTest.js`; } - const w = new Worker(testPath, { argv: process.argv.slice(2), }); diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts index b0ff2d1d5..9b7ecf409 100644 --- a/internal-tooling/performanceUtils.ts +++ b/internal-tooling/performanceUtils.ts @@ -32,6 +32,7 @@ export const OUTPUT_FORMATS = { export const PERFORMANCE_TEST_TYPES = { WRITE_ONE_READ_THREE: 'w1r3', RANGE_READ: 'range-read', + MANY_SMALL: 'many-small', TRANSFER_MANAGER_UPLOAD_MANY_FILES: 'tm-upload', TRANSFER_MANAGER_DOWNLOAD_MANY_FILES: 'tm-download', TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD: 'tm-chunked',