Merge pull request #5632 from nimrod-becker/nimrod_backport_5_0

Backport to 5.0.1

config.js

@@ -188,8 +188,7 @@ config.MIRROR_WRITER_EMPTY_DELAY = 30000;
 config.MIRROR_WRITER_MARKER_STORE_PERIOD = 10 * 60000; // store markers every 10 min
 
 config.TIER_TTF_WORKER_ENABLED = true;
-config.TIER_TTF_WORKER_BATCH_SIZE = 50;
-config.TIER_TTF_WORKER_BATCH_DELAY = 50;
+config.TIER_TTF_WORKER_BATCH_DELAY = 500;
 config.TIER_TTF_WORKER_EMPTY_DELAY = 30000;
 
 config.TIER_SPILLBACK_WORKER_ENABLED = true;

frontend/src/app/reducers/host-parts-reducer.js

@@ -82,15 +82,13 @@ function onFailFetchHostObjects(state, { payload }) {
 
 function onFetchCloudResourceObjects(state, { payload }) {
     return onFetchHostObjects(
-        state,
-        { payload: _resourceQueryToHostQuery(payload) }
+        state, { payload: _resourceQueryToHostQuery(payload) }
     );
 }
 
 function onCompleteFetchCloudResourceObjects(state, { payload }) {
     return onCompleteFetchHostObjects(
-        state,
-        {
+        state, {
             payload: {
                 query: _resourceQueryToHostQuery(payload.query),
                 response: payload.response
@@ -101,8 +99,7 @@ function onCompleteFetchCloudResourceObjects(state, { payload }) {
 
 function onFailCloudResourceObjects(state, { payload }) {
     return onFailFetchHostObjects(
-        state,
-        {
+        state, {
             payload: {
                 query: _resourceQueryToHostQuery(payload.query),
                 error: payload.error

package.json

@@ -1,6 +1,6 @@
 {
   "name": "noobaa-core",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "private": true,
   "license": "SEE LICENSE IN LICENSE",
   "description": "",
@@ -66,6 +66,8 @@
     "ajv": "6.10.0",
     "aws-sdk": "2.488.0",
     "azure-storage": "2.10.3",
+    "babel-plugin-transform-runtime": "6.23.0",
+    "babel-preset-es2015": "6.24.1",
     "bcrypt": "3.0.6",
     "big-integer": "1.6.44",
     "bindings": "1.5.0",
@@ -88,7 +90,7 @@
     "ip": "1.1.5",
     "jsonwebtoken": "8.5.1",
     "linux-blockutils": "0.1.0",
-    "lodash": "4.17.11",
+    "lodash": "4.17.14",
     "mime": "2.4.4",
     "minimist": "1.2.0",
     "mkdirp": "0.5.1",
@@ -97,7 +99,7 @@
     "mongodb": "3.2.7",
     "mongodb-uri": "0.9.7",
     "morgan": "1.9.1",
-    "multer": "1.4.1",
+    "multer": "1.4.2",
     "nan": "2.14.0",
     "ncp": "2.0.0",
     "net-ping": "1.2.3",
@@ -110,24 +112,22 @@
     "request": "2.88.0",
     "rimraf": "2.6.3",
     "rotating-file-stream": "1.4.2",
+    "seedrandom": "3.0.1",
     "serve-favicon": "2.5.0",
     "setimmediate": "1.0.5",
     "sinon": "7.3.2",
-    "ssh2": "0.8.4",
+    "ssh2": "0.8.5",
     "ts-node": "8.3.0",
     "tslint": "5.16.0",
-    "typescript": "3.5.1",
+    "typescript": "3.5.3",
     "utf-8-validate": "5.0.2",
     "uuid": "3.3.2",
     "why-is-node-running": "2.1.0",
     "ws": "7.0.1",
     "xml2js": "0.4.19",
     "yaml": "1.6.0",
     "yauzl": "2.10.0",
-    "yazl": "2.5.1",
-    "seedrandom": "3.0.1",
-    "babel-plugin-transform-runtime": "6.23.0",
-    "babel-preset-es2015": "6.24.1"
+    "yazl": "2.5.1"
   },
   "devDependencies": {
     "mocha": "6.1.4",

src/agent/block_store_services/block_store_fs.js

@@ -11,7 +11,6 @@ const fs_utils = require('../../util/fs_utils');
 const os_utils = require('../../util/os_utils');
 const config = require('../../../config.js');
 const string_utils = require('../../util/string_utils');
-const promise_utils = require('../../util/promise_utils');
 const BlockStoreBase = require('./block_store_base').BlockStoreBase;
 const { RpcError } = require('../../rpc');
 
@@ -42,7 +41,6 @@ class BlockStoreFs extends BlockStoreBase {
         return P.map(dir_list, dir => fs_utils.create_path(dir), {
                 concurrency: 10
             })
-            .then(() => this._upgrade_to_blocks_tree())
             .then(() => fs.statAsync(this.usage_path)
                 .catch(ignore_not_found)
             )
@@ -263,98 +261,6 @@ class BlockStoreFs extends BlockStoreBase {
         return path.join(this.blocks_path_root, block_dir, file);
     }
 
-    _upgrade_to_blocks_tree() {
-        return fs.statAsync(this.old_blocks_path)
-            .catch(err => {
-                // when it doesn't exist it means we don't need to upgrade
-                // on any other error, we ignore as we don't really expect
-                // any error we have anything to do about it
-                if (err.code !== 'ENOENT') {
-                    dbg.log0('_upgrade_to_blocks_tree:',
-                        'Old blocks dir failed to stat, ignoring',
-                        this.old_blocks_path, err);
-                }
-            })
-            .then(stat => {
-                if (!stat) return;
-                if (stat.size > 64 * 1024 * 1024) {
-                    dbg.warn('_upgrade_to_blocks_tree:',
-                        'Old blocks dir is huge and might crash the process',
-                        'spawning upgrade_agent_to_blocks_tree.py to the rescue',
-                        this.old_blocks_path, stat);
-                    // spawning the python script to iterativly move
-                    // the large blocks to blocks tree.
-                    // the output of the script will be redirected to our stdout
-                    // though this will not be logged through our debug module,
-                    // but still collected in diagnostics.
-                    return promise_utils.spawn('python', [
-                        'src/agent/upgrade_agent_to_blocks_tree.py',
-                        '--wet',
-                        this.root_path
-                    ]);
-                }
-                if (stat.size > 8 * 1024 * 1024) {
-                    dbg.warn('_upgrade_to_blocks_tree:',
-                        'Old blocks dir is pretty big and might take longer to read',
-                        this.old_blocks_path, stat);
-                }
-                return this._move_to_blocks_tree();
-            });
-    }
-
-    _move_to_blocks_tree() {
-        let num_move_errors = 0;
-        dbg.log0('_upgrade_to_blocks_tree: reading', this.old_blocks_path);
-        return fs.readdirAsync(this.old_blocks_path)
-            .then(files => {
-                dbg.log2('found', files.length, 'files to move. files:', files);
-                return P.map(files, file => {
-                    let file_split = file.split('.');
-                    let new_path = this._get_block_other_path(file);
-                    if (file_split.length === 2) {
-                        let block_id = file_split[0];
-                        let suffix = file_split[1];
-                        if (suffix === 'data') {
-                            new_path = this._get_block_data_path(block_id);
-                        } else if (suffix === 'meta') {
-                            new_path = this._get_block_meta_path(block_id);
-                        }
-                    }
-                    let old_path = path.join(this.old_blocks_path, file);
-                    return fs.renameAsync(old_path, new_path)
-                        .catch(err => {
-                            // we log the error here and count, but do not throw
-                            // to try and move all the rest of the files.
-                            num_move_errors += 1;
-                            dbg.error('_upgrade_to_blocks_tree:',
-                                'failed moving', old_path, '->', new_path, err);
-                        });
-                }, {
-                    // limit the number of promises to use for moving blocks
-                    // - set arbitrarily for now
-                    concurrency: 10
-                });
-            })
-            .then(() => fs.rmdirAsync(this.old_blocks_path))
-            .then(() => {
-                // since we also successfuly deleted the old blocks dir
-                // it must mean there are no leftovers in anycase.
-                // so even if we counted num_move_errors, it might have been
-                // due to parallel operations with another process,
-                // so we ignore it.
-                if (num_move_errors) {
-                    dbg.log0('_upgrade_to_blocks_tree: finished', this.old_blocks_path,
-                        'eventhough we had num_move_errors', num_move_errors);
-                }
-                dbg.log0('_upgrade_to_blocks_tree: done', this.old_blocks_path);
-            })
-            .catch(err => {
-                dbg.error('_upgrade_to_blocks_tree: failed',
-                    this.old_blocks_path, 'num_move_errors', num_move_errors,
-                    err.stack || err);
-            });
-    }
-
 }
 
 function ignore_not_found(err) {

src/api/api.js

@@ -29,7 +29,6 @@ api_schema.register_api(require('./tiering_policy_api'));
 api_schema.register_api(require('./pool_api'));
 api_schema.register_api(require('./cluster_server_api'));
 api_schema.register_api(require('./cluster_internal_api'));
-api_schema.register_api(require('./upgrade_api'));
 api_schema.register_api(require('./server_inter_process_api'));
 api_schema.register_api(require('./hosted_agents_api'));
 api_schema.register_api(require('./frontend_notifications_api'));
@@ -64,7 +63,6 @@ class APIClient {
         this.pool = undefined;
         this.cluster_server = undefined;
         this.cluster_internal = undefined;
-        this.upgrade = undefined;
         this.server_inter_process = undefined;
         this.hosted_agents = undefined;
         this.frontend_notifications = undefined;

src/api/cluster_internal_api.js

@@ -272,15 +272,6 @@ module.exports = {
             }
         },
 
-        // used for backward compatability with basic_server_ops upgrade during tests
-        // TODO: remove after we don't use versions with upgrade in cluster_server
-        upgrade_cluster: {
-            method: 'POST',
-            auth: {
-                system: 'admin',
-            }
-        },
-
     },
 
     definitions: {