test: add ability to test many files without transfer manager

googleapis · Nov 8, 2023 · 102b749 · 102b749
1 parent eb772c9
commit 102b749
Show file tree

Hide file tree

Showing 3 changed files with 101 additions and 6 deletions.
diff --git a/internal-tooling/performPerformanceTest.ts b/internal-tooling/performPerformanceTest.ts
@@ -30,7 +30,7 @@ import {
   PERFORMANCE_TEST_TYPES,
   TestResult,
 } from './performanceUtils.js';
-import {Bucket} from '../src/index.js';
+import {Bucket, File} from '../src/index.js';
 import {getDirName} from '../src/util.js';
 
 const TEST_NAME_STRING = 'nodejs-perf-metrics';
@@ -64,6 +64,9 @@ async function main() {
     case PERFORMANCE_TEST_TYPES.RANGE_READ:
       results = await performRangedReadTest();
       break;
+    case PERFORMANCE_TEST_TYPES.MANY_SMALL:
+      results = await performManyFilesTest();
+      break;
     default:
       break;
   }
@@ -214,4 +217,94 @@ async function performWriteReadTest(): Promise<TestResult[]> {
   return results;
 }
 
+/**
+ * Performs an iteration of the many files test.
+ *
+ * @returns {Promise<TestResult[]>} Promise that resolves to an array of test results for the iteration.
+ */
+async function performManyFilesTest(): Promise<TestResult[]> {
+  const results: TestResult[] = [];
+  const fileSizeRange = getLowHighFileSize(argv.object_size as string);
+  const fileMap = new Map<string, File>();
+  let totalSizeInBytes = 0;
+
+  for (let i = 0; i < (argv.num_objects as number); i++) {
+    const fileName = generateRandomFileName(TEST_NAME_STRING);
+    const file = bucket.file(`${fileName}`);
+    totalSizeInBytes += generateRandomFile(
+      fileName,
+      fileSizeRange.low,
+      fileSizeRange.high,
+      getDirName()
+    );
+    fileMap.set(fileName, file);
+  }
+
+  let iterationResult: TestResult = {
+    op: 'WRITE',
+    objectSize: totalSizeInBytes,
+    appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES,
+    crc32cEnabled: checkType === 'crc32c',
+    md5Enabled: checkType === 'md5',
+    api: 'JSON',
+    elapsedTimeUs: 0,
+    cpuTimeUs: -1,
+    status: 'OK',
+    chunkSize: totalSizeInBytes,
+    workers: argv.workers as number,
+    library: 'nodejs',
+    transferSize: totalSizeInBytes,
+    transferOffset: 0,
+    bucketName: bucket.name,
+  };
+
+  let start = performance.now();
+  for (const curFileName of fileMap.keys()) {
+    await bucket.upload(`${getDirName()}/${curFileName}`, {
+      validation: checkType,
+    });
+  }
+  let end = performance.now();
+  iterationResult.elapsedTimeUs = Math.round((end - start) * 1000);
+  results.push(iterationResult);
+
+  iterationResult = {
+    op: 'READ[0]',
+    objectSize: totalSizeInBytes,
+    appBufferSize: NODE_DEFAULT_HIGHWATER_MARK_BYTES,
+    crc32cEnabled: checkType === 'crc32c',
+    md5Enabled: checkType === 'md5',
+    api: 'JSON',
+    elapsedTimeUs: 0,
+    cpuTimeUs: -1,
+    status: 'OK',
+    chunkSize: totalSizeInBytes,
+    workers: argv.workers as number,
+    library: 'nodejs',
+    transferSize: totalSizeInBytes,
+    transferOffset: 0,
+    bucketName: bucket.name,
+  };
+
+  for (let i = 0; i < DEFAULT_NUMBER_OF_READS; i++) {
+    start = performance.now();
+    for (const curToDownload of fileMap.values()) {
+      await curToDownload.download({
+        validation: checkType,
+        destination: path.join(getDirName(), curToDownload.name),
+      });
+    }
+    end = performance.now();
+  }
+
+  iterationResult.elapsedTimeUs = Math.round((end - start) * 1000);
+  results.push(iterationResult);
+  for (const curToDelete of fileMap.values()) {
+    await curToDelete.delete({ignoreNotFound: true});
+    cleanupFile(curToDelete.name, getDirName());
+  }
+
+  return results;
+}
+
 main();
diff --git a/internal-tooling/performanceTest.ts b/internal-tooling/performanceTest.ts
@@ -60,6 +60,7 @@ function main() {
  * When the worker passes back the results, they are appended to the results file.
  */
 function createWorker() {
+  const dirName = getDirName().replace('/src', '/internal-tooling');
   iterationsRemaining--;
   log(
     `Starting new iteration. Current iterations remaining: ${iterationsRemaining}`,
@@ -68,9 +69,10 @@ function createWorker() {
   let testPath = '';
   if (
     argv.test_type === PERFORMANCE_TEST_TYPES.WRITE_ONE_READ_THREE ||
-    argv.test_type === PERFORMANCE_TEST_TYPES.RANGE_READ
+    argv.test_type === PERFORMANCE_TEST_TYPES.RANGE_READ ||
+    argv.test_type === PERFORMANCE_TEST_TYPES.MANY_SMALL
   ) {
-    testPath = `${getDirName()}/performPerformanceTest.js`;
+    testPath = `${dirName}/performPerformanceTest.js`;
   } else if (
     argv.test_type ===
       PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_UPLOAD_MANY_FILES ||
@@ -79,17 +81,16 @@ function createWorker() {
     argv.test_type ===
       PERFORMANCE_TEST_TYPES.TRANSFER_MANAGER_DOWNLOAD_MANY_FILES
   ) {
-    testPath = `${getDirName()}/performTransferManagerTest.js`;
+    testPath = `${dirName}/performTransferManagerTest.js`;
   } else if (
     argv.test_type ===
       PERFORMANCE_TEST_TYPES.APPLICATION_UPLOAD_MULTIPLE_OBJECTS ||
     argv.test_type === PERFORMANCE_TEST_TYPES.APPLICATION_LARGE_FILE_DOWNLOAD ||
     argv.test_type ===
       PERFORMANCE_TEST_TYPES.APPLICATION_DOWNLOAD_MULTIPLE_OBJECTS
   ) {
-    testPath = `${getDirName()}/performApplicationPerformanceTest.js`;
+    testPath = `${dirName}/performApplicationPerformanceTest.js`;
   }
-
   const w = new Worker(testPath, {
     argv: process.argv.slice(2),
   });

diff --git a/internal-tooling/performanceUtils.ts b/internal-tooling/performanceUtils.ts
@@ -32,6 +32,7 @@ export const OUTPUT_FORMATS = {
 export const PERFORMANCE_TEST_TYPES = {
   WRITE_ONE_READ_THREE: 'w1r3',
   RANGE_READ: 'range-read',
+  MANY_SMALL: 'many-small',
   TRANSFER_MANAGER_UPLOAD_MANY_FILES: 'tm-upload',
   TRANSFER_MANAGER_DOWNLOAD_MANY_FILES: 'tm-download',
   TRANSFER_MANAGER_CHUNKED_FILE_DOWNLOAD: 'tm-chunked',