Merge pull request #7963 from nimrod-becker/backport_to_5_15

Backport to 5.15

src/agent/block_store_services/block_store_client.js

@@ -23,7 +23,7 @@ const block_store_info_cache = new LRUCache({
     max_usage: 1000,
     expiry_ms: 2 * 60 * 1000,
     make_key: params => params.options.address,
-    load: async ({ rpc_client, options }) => rpc_client.block_store.get_block_store_info(null, options),
+    load: async ({ rpc_client, options }) => rpc_client.block_store.get_block_store_info({}, options),
 });
 class BlockStoreClient {
 
@@ -98,8 +98,8 @@ class BlockStoreClient {
                         }
                     },
                 });
+                dbg.log3('writing block id to gcp: ', block_id);
                 await buffer_utils.write_to_stream(write_stream, data);
-                write_stream.end();
                 const data_length = data.length;
                 const usage = data_length ? {
                     size: (block_md.is_preallocated ? 0 : data_length) + encoded_md.length,

src/agent/block_store_services/block_store_google.js

@@ -165,7 +165,6 @@ class BlockStoreGoogle extends BlockStoreBase {
         dbg.log3('writing block id to cloud: ', key);
         try {
             await buffer_utils.write_to_stream(write_stream, data);
-            write_stream.end();
             const usage = {
                 size: data.length + encoded_md.length,
                 count: 1

src/sdk/bucketspace_s3.js

@@ -35,6 +35,7 @@ class BucketSpaceS3 {
             const buckets_with_name_change = buckets.map(b => ({ name: new SensitiveString(b.Name) }));
             return { buckets_with_name_change };
         } catch (err) {
+            noobaa_s3_client.fix_error_object(err);
             this._translate_error_code(err);
             throw err;
         }
@@ -62,6 +63,7 @@ class BucketSpaceS3 {
                 undeletable: 'NOT_EMPTY',
             };
         } catch (err) {
+            noobaa_s3_client.fix_error_object(err);
             this._translate_error_code(err);
             throw err;
         }
@@ -73,6 +75,7 @@ class BucketSpaceS3 {
             console.log(`bss3: create_bucket ${name}`);
             await this.s3.createBucket({ Bucket: name });
         } catch (err) {
+            noobaa_s3_client.fix_error_object(err);
             this._translate_error_code(err);
             throw err;
         }
@@ -84,6 +87,7 @@ class BucketSpaceS3 {
             console.log(`bss3: delete_fs_bucket ${name}`);
             await this.s3.deleteBucket({ Bucket: name });
         } catch (err) {
+            noobaa_s3_client.fix_error_object(err);
             this._translate_error_code(err);
             throw err;
         }

src/sdk/noobaa_s3_client/noobaa_s3_client.js

@@ -100,14 +100,9 @@ function get_requestHandler_with_suitable_agent(endpoint) {
     }
 }
 
-// v2: code, v3: Code
-function check_error_code(err, code) {
-    return err.code === code || err.Code === code;
-}
-
 /**
  * When using aws sdk v3 we get the error object with values that start with an uppercase instead of lowercase
- * we want to fix this value to be start with a lowercase.
+ * we want to fix this value to be start with a lowercase (example: v2: code, v3: Code).
  * 
  * @param {object} err
  */
@@ -134,6 +129,5 @@ function fix_error_object(err) {
 exports.get_s3_client_v3_params = get_s3_client_v3_params;
 exports.change_s3_client_params_to_v2_structure = change_s3_client_params_to_v2_structure;
 exports.get_sdk_class_str = get_sdk_class_str;
-exports.check_error_code = check_error_code;
 exports.fix_error_object = fix_error_object;
 exports.get_requestHandler_with_suitable_agent = get_requestHandler_with_suitable_agent;

src/server/bg_services/namespace_monitor.js

@@ -142,12 +142,12 @@ class NamespaceMonitor {
                 Key: block_key
             });
         } catch (err) {
-            noobaa_s3_client.fix_error_object(err); //This makes the noobaa_s3_client.check_error_code redundant, we should consider removing it.
-            if (noobaa_s3_client.check_error_code(err, 'AccessDenied') && nsr.is_readonly_namespace()) {
+            noobaa_s3_client.fix_error_object(err);
+            if (err.code === 'AccessDenied' && nsr.is_readonly_namespace()) {
                 return;
             }
             dbg.log1('test_s3_resource: got error:', err);
-            if (!noobaa_s3_client.check_error_code(err, 'NoSuchKey')) throw err;
+            if (err.code !== 'NoSuchKey') throw err;
         }
 
     }

src/server/system_services/bucket_server.js

@@ -1208,6 +1208,7 @@ async function get_cloud_buckets(req) {
             return buckets.map(bucket => _inject_usage_to_cloud_bucket(bucket.Name, connection.endpoint, used_cloud_buckets));
         }
     } catch (err) {
+        noobaa_s3_client.fix_error_object(err); // only relevant when using AWS SDK v3
         if (err instanceof P.TimeoutError) {
             dbg.log0('failed reading (t/o) external buckets list', req.rpc_params);
         } else {

src/tools/diagnostics/analyze_resource/analyze_resource_gcp.js

@@ -4,6 +4,7 @@
 const { inspect } = require('util');
 const { Storage } = require('@google-cloud/storage');
 const dbg = require('../../../util/debug_module')(__filename);
+const buffer_utils = require('../../../util/buffer_utils');
 dbg.set_process_name('analyze_resource');
 const CloudVendor = require('./analyze_resource_cloud_vendor_abstract');
 
@@ -58,8 +59,7 @@ class AnalyzeGcp extends CloudVendor {
             .bucket(bucket)
             .file(key)
             .createWriteStream();
-        stream.write(''); //write an empty file
-        stream.end();
+        await buffer_utils.write_to_stream(stream, ''); //write an empty file
         stream.on('response', resp => {
             dbg.log0(`Write of ${key} response: ${inspect(resp)}`);
         });

src/util/buffer_utils.js

@@ -167,11 +167,12 @@ function count_length(buffers) {
 function write_to_stream(writable, buf) {
     return new Promise((resolve, reject) => {
         writable.once('error', reject);
+        writable.once('finish', resolve);
         writable.write(buf, err => {
             if (err) {
                 return reject(err);
             }
-            return resolve();
+            writable.end();
         });
     });
 }