diff --git a/package.json b/package.json index aa34c6f67b..09fcedf691 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "noobaa-core", - "version": "4.0.1", + "version": "4.0.2", "private": true, "license": "SEE LICENSE IN LICENSE", "description": "", diff --git a/src/api/stats_api.js b/src/api/stats_api.js index 94f811df60..08b9ed9fe0 100644 --- a/src/api/stats_api.js +++ b/src/api/stats_api.js @@ -33,6 +33,16 @@ module.exports = { } }, + get_partial_accounts_stats: { + method: 'GET', + reply: { + $ref: '#/definitions/partial_accounts_stats' + }, + auth: { + system: 'admin' + } + }, + get_nodes_stats: { method: 'GET', reply: { @@ -483,7 +493,7 @@ module.exports = { partial_stats: { type: 'object', - required: ['systems_stats', 'cloud_pool_stats'], + required: ['systems_stats', 'cloud_pool_stats', 'accounts_stats'], properties: { systems_stats: { $ref: '#/definitions/partial_systems_stats' @@ -491,6 +501,9 @@ module.exports = { cloud_pool_stats: { $ref: '#/definitions/cloud_pool_stats' }, + accounts_stats: { + $ref: '#/definitions/partial_accounts_stats' + }, } }, @@ -502,20 +515,61 @@ module.exports = { type: 'array', items: { type: 'object', - required: ['name', 'free_space', 'total_space', 'buckets_stats'], + required: ['name', 'capacity', 'reduction_ratio', 'savings', 'buckets_stats', 'usage_by_project', 'usage_by_bucket_class'], properties: { name: { type: 'string' }, - free_space: { - $ref: 'common_api#/definitions/bigint' + capacity: { + type: 'number' }, - total_space: { - $ref: 'common_api#/definitions/bigint' + savings: { + type: 'number' + }, + reduction_ratio: { + type: 'number' }, buckets_stats: { $ref: '#/definitions/partial_buckets_stats' }, + usage_by_project: { + type: 'object', + additionalProperties: true, + properties: {}, + }, + usage_by_bucket_class: { + type: 'object', + additionalProperties: true, + properties: {}, + }, + } + } + } + } + }, + + partial_accounts_stats: { + type: 'object', + required: ['accounts'], + properties: { + accounts: { + type: 'array', + items: { + type: 'object', + required: ['account', 'read_count', 'write_count', 'read_write_bytes'], + properties: { + account: { + type: 'string' + }, + read_count: { + type: 'number' + }, + write_count: { + type: 'number' + }, + read_write_bytes: { + type: 'number' + }, } } } diff --git a/src/deploy/NVA_build/noobaa_core.yaml b/src/deploy/NVA_build/noobaa_core.yaml index f7dcd360b6..3e9ee6d5fd 100644 --- a/src/deploy/NVA_build/noobaa_core.yaml +++ b/src/deploy/NVA_build/noobaa_core.yaml @@ -126,7 +126,7 @@ spec: tcpSocket: port: 6001 timeoutSeconds: 5 - image: noobaa/noobaa-core:4.0.1 + image: noobaa/noobaa-core:4.0.2 imagePullPolicy: IfNotPresent resources: # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ diff --git a/src/server/analytic_services/prometheus_reporting.js b/src/server/analytic_services/prometheus_reporting.js index f17a809f3f..0c7353a5e5 100644 --- a/src/server/analytic_services/prometheus_reporting.js +++ b/src/server/analytic_services/prometheus_reporting.js @@ -17,6 +17,30 @@ const METRIC_RECORDS = Object.freeze([{ help: 'Cloud Resource Types in the System', labelNames: ['type', 'count'] } +}, { + metric_type: 'Gauge', + metric_variable: 'projects_capacity_usage', + configuration: { + name: get_metric_name('projects_capacity_usage'), + help: 'Projects Capacity Usage', + labelNames: ['project', 'count'] + } +}, { + metric_type: 'Gauge', + metric_variable: 'accounts_io_usage', + configuration: { + name: get_metric_name('accounts_io_usage'), + help: 'Accounts I/O Usage', + labelNames: ['account', 'read_count', 'write_count'] + } +}, { + metric_type: 'Gauge', + metric_variable: 'bucket_class_capacity_usage', + configuration: { + name: get_metric_name('bucket_class_capacity_usage'), + help: 'Bucket Class Capacity Usage', + labelNames: ['bucket_class', 'count'] + } }, { metric_type: 'Gauge', metric_variable: 'unhealthy_cloud_types', @@ -105,6 +129,22 @@ const METRIC_RECORDS = Object.freeze([{ help: 'Objects On Object Bucket Claims', }, generate_default_set: true, +}, { + metric_type: 'Gauge', + metric_variable: 'reduction_ratio', + configuration: { + name: get_metric_name('reduction_ratio'), + help: 'Object Efficiency Ratio', + }, + generate_default_set: true, +}, { + metric_type: 'Gauge', + metric_variable: 'object_savings', + configuration: { + name: get_metric_name('object_savings'), + help: 'Object Savings', + }, + generate_default_set: true, }]); @@ -173,6 +213,31 @@ class PrometheusReporting { this._metrics.unhealthy_cloud_types.set({ type: 'S3_Compatible' }, types.unhealthy_cloud_pool_target.s3_comp_unhealthy); } + set_bucket_class_capacity_usage(usage_info) { + if (!this.enabled()) return; + this._metrics.bucket_class_capacity_usage.reset(); + for (let [key, value] of Object.entries(usage_info)) { + this._metrics.bucket_class_capacity_usage.set({ bucket_class: key }, value); + } + } + + set_projects_capacity_usage(usage_info) { + if (!this.enabled()) return; + this._metrics.projects_capacity_usage.reset(); + for (let [key, value] of Object.entries(usage_info)) { + this._metrics.projects_capacity_usage.set({ project: key }, value); + } + } + + set_accounts_io_usage(accounts_info) { + if (!this.enabled()) return; + this._metrics.accounts_io_usage.reset(); + accounts_info.accounts.forEach(account_info => { + const { account, read_count, write_count, read_write_bytes } = account_info; + this._metrics.accounts_io_usage.set({ account, read_count, write_count }, read_write_bytes); + }); + } + set_object_sizes(sizes) { if (!this.enabled()) return; for (const bin of sizes) { diff --git a/src/server/system_services/account_server.js b/src/server/system_services/account_server.js index 89cfe83cca..daa39b2111 100644 --- a/src/server/system_services/account_server.js +++ b/src/server/system_services/account_server.js @@ -706,31 +706,26 @@ function check_external_connection(req) { return P.resolve() .then(() => { switch (endpoint_type) { - case 'AZURE': - { - return check_azure_connection(params); - } + case 'AZURE': { + return check_azure_connection(params); + } case 'AWS': case 'S3_COMPATIBLE': - case 'FLASHBLADE': - { - return check_aws_connection(params); - } + case 'FLASHBLADE': { + return check_aws_connection(params); + } - case 'NET_STORAGE': - { - return check_net_storage_connection(params); - } - case 'GOOGLE': - { - return check_google_connection(params); - } + case 'NET_STORAGE': { + return check_net_storage_connection(params); + } + case 'GOOGLE': { + return check_google_connection(params); + } - default: - { - throw new Error('Unknown endpoint type'); - } + default: { + throw new Error('Unknown endpoint type'); + } } }); } diff --git a/src/server/system_services/stats_aggregator.js b/src/server/system_services/stats_aggregator.js index 1c7078a6d3..e416e5b586 100644 --- a/src/server/system_services/stats_aggregator.js +++ b/src/server/system_services/stats_aggregator.js @@ -19,6 +19,7 @@ const nodes_client = require('../node_services/nodes_client'); const system_store = require('../system_services/system_store').get_instance(); const system_server = require('../system_services/system_server'); const bucket_server = require('../system_services/bucket_server'); +const account_server = require('../system_services/account_server'); const object_server = require('../object_services/object_server'); const auth_server = require('../common_services/auth_server'); const server_rpc = require('../server_rpc'); @@ -106,11 +107,29 @@ const PARTIAL_SYSTEM_STATS_DEFAULTS = { systems: [], }; +const PARTIAL_ACCOUNT_IO_STATS = { + accounts: [], +}; + +const PARTIAL_SINGLE_ACCOUNT_DEFAULTS = { + account: '', + read_count: 0, + write_count: 0, + read_write_bytes: 0, +}; + const PARTIAL_SINGLE_SYS_DEFAULTS = { name: '', - free_space: 0, - total_space: 0, + capacity: 0, + reduction_ratio: 0, + savings: 0, buckets_stats: PARTIAL_BUCKETS_STATS_DEFAULTS, + usage_by_project: { + Others: 0, + }, + usage_by_bucket_class: { + Others: 0, + }, }; @@ -206,6 +225,39 @@ async function get_systems_stats(req) { } } +async function get_partial_accounts_stats(req) { + const accounts_stats = _.cloneDeep(PARTIAL_ACCOUNT_IO_STATS); + try { + // TODO: Either make a large query or per account + // In case of large query we also need to set a limit and tirgger listing queries so we won't crash + accounts_stats.accounts = await P.all(_.map(system_store.data.accounts, async account => { + const new_req = _.defaults({ + rpc_params: { accounts: [account.email], from: new Date(0), till: new Date() }, + }, req); + + const account_usage_info = await account_server.get_account_usage(new_req); + if (_.isEmpty(account_usage_info)) return; + + const { read_count, write_count } = account_usage_info[0]; + const read_bytes = size_utils.json_to_bigint(account_usage_info[0].read_bytes || size_utils.BigInteger.zero); + const write_bytes = size_utils.json_to_bigint(account_usage_info[0].write_bytes || size_utils.BigInteger.zero); + const read_write_bytes = read_bytes.plus(write_bytes).toJSNumber(); + return _.defaults({ + account: account.email.unwrap(), + read_count, + write_count, + read_write_bytes, + }, PARTIAL_SINGLE_ACCOUNT_DEFAULTS); + })); + accounts_stats.accounts = _.compact(accounts_stats.accounts); + return accounts_stats; + } catch (err) { + dbg.warn('Error in collecting partial account i/o stats,', + 'skipping current sampling point', err.stack || err); + throw err; + } +} + async function get_partial_systems_stats(req) { const sys_stats = _.cloneDeep(PARTIAL_SYSTEM_STATS_DEFAULTS); @@ -215,7 +267,13 @@ async function get_partial_systems_stats(req) { system: system }, req); - const { buckets_stats, objects_sys } = await _partial_buckets_info(new_req); + const { + buckets_stats, + objects_sys, + objects_non_namespace_buckets_sys, + usage_by_bucket_class, + usage_by_project, + } = await _partial_buckets_info(new_req); // nodes - count, online count, allocated/used storage aggregate by pool const nodes_aggregate_pool_with_cloud_no_mongo = await nodes_client.instance() @@ -225,11 +283,24 @@ async function get_partial_systems_stats(req) { used: objects_sys.size, }, nodes_aggregate_pool_with_cloud_no_mongo.storage, SYS_STORAGE_DEFAULTS)); + const { chunks_capacity, logical_size } = objects_non_namespace_buckets_sys; + const chunks = size_utils.bigint_to_bytes(chunks_capacity); + const logical = size_utils.bigint_to_bytes(logical_size); + const reduction_ratio = (logical / chunks) || 1; + const savings = logical_size.minus(chunks_capacity).toJSNumber(); + + const free_bytes = size_utils.bigint_to_bytes(storage.free); + const total_bytes = size_utils.bigint_to_bytes(storage.total); + const capacity = 100 - Math.floor(((free_bytes / total_bytes) || 1) * 100); + return _.defaults({ name: system.name, - total_space: storage.total, - free_space: storage.free, - buckets_stats + capacity, + reduction_ratio, + savings, + buckets_stats, + usage_by_bucket_class, + usage_by_project }, PARTIAL_SINGLE_SYS_DEFAULTS); })); return sys_stats; @@ -244,9 +315,20 @@ async function get_partial_systems_stats(req) { async function _partial_buckets_info(req) { const buckets_stats = _.cloneDeep(PARTIAL_BUCKETS_STATS_DEFAULTS); const objects_sys = { - count: size_utils.BigInteger.zero, size: size_utils.BigInteger.zero, + count: 0, + }; + const objects_non_namespace_buckets_sys = { + chunks_capacity: size_utils.BigInteger.zero, + logical_size: size_utils.BigInteger.zero, }; + const usage_by_project = { + Others: size_utils.BigInteger.zero, + }; + const usage_by_bucket_class = { + Others: size_utils.BigInteger.zero, + }; + try { for (const bucket of system_store.data.buckets) { if (String(bucket.system._id) !== String(req.system._id)) return; @@ -257,13 +339,32 @@ async function _partial_buckets_info(req) { const bucket_info = await bucket_server.read_bucket(new_req); objects_sys.size = objects_sys.size.plus( - (bucket_info.storage_stats && bucket_info.storage_stats.objects_size) || 0 + (bucket.storage_stats && size_utils.json_to_bigint(bucket.storage_stats.objects_size)) || 0 ); - objects_sys.count = objects_sys.count.plus(bucket_info.num_objects || 0); + objects_sys.count += bucket_info.num_objects || 0; if (bucket_info.namespace) return; + objects_non_namespace_buckets_sys.chunks_capacity = objects_non_namespace_buckets_sys.chunks_capacity.plus( + (bucket.storage_stats && size_utils.json_to_bigint(bucket.storage_stats.chunks_capacity)) || 0 + ); + + objects_non_namespace_buckets_sys.logical_size = objects_non_namespace_buckets_sys.logical_size.plus( + (bucket.storage_stats && size_utils.json_to_bigint(bucket.storage_stats.objects_size)) || 0 + ); + + const bucket_project = (bucket_info.bucket_claim && bucket_info.bucket_claim.namespace) || 'Others'; + const existing_project = usage_by_project[bucket_project] || size_utils.BigInteger.zero; + usage_by_project[bucket_project] = existing_project.plus( + (bucket.storage_stats && size_utils.json_to_bigint(bucket.storage_stats.objects_size)) || 0 + ); + const bucket_class = (bucket_info.bucket_claim && bucket_info.bucket_claim.policy_type) || 'Others'; + const existing_bucket_class = usage_by_bucket_class[bucket_class] || size_utils.BigInteger.zero; + usage_by_bucket_class[bucket_class] = existing_bucket_class.plus( + (bucket.storage_stats && size_utils.json_to_bigint(bucket.storage_stats.objects_size)) || 0 + ); + buckets_stats.buckets += 1; buckets_stats.objects_in_buckets += bucket_info.num_objects; @@ -277,9 +378,17 @@ async function _partial_buckets_info(req) { 'HIGH_DATA_ACTIVITY', 'OPTIMAL', ]; - if (!_.includes(OPTIMAL_MODES, bucket_info.mode)) buckets_stats.unhealthy_buckets += 1; + if (!!_.includes(OPTIMAL_MODES, bucket_info.mode)) buckets_stats.unhealthy_buckets += 1; } - return { buckets_stats, objects_sys }; + + Object.keys(usage_by_bucket_class).forEach(key => { + usage_by_bucket_class[key] = usage_by_bucket_class[key].toJSNumber(); + }); + Object.keys(usage_by_project).forEach(key => { + usage_by_project[key] = usage_by_project[key].toJSNumber(); + }); + + return { buckets_stats, objects_sys, objects_non_namespace_buckets_sys, usage_by_project, usage_by_bucket_class }; } catch (err) { dbg.warn('Error in collecting partial buckets stats,', 'skipping current sampling point', err.stack || err); @@ -446,30 +555,26 @@ async function get_cloud_pool_stats(req) { cloud_pool_stats.cloud_pool_count += 1; switch (pool.cloud_pool_info.endpoint_type) { case 'AWS': - if (_.includes(OPTIMAL_MODES, pool_info.mode)) { - cloud_pool_stats.cloud_pool_target.amazon += 1; - } else { + cloud_pool_stats.cloud_pool_target.amazon += 1; + if (!_.includes(OPTIMAL_MODES, pool_info.mode)) { cloud_pool_stats.unhealthy_cloud_pool_target.amazon_unhealthy += 1; } break; case 'AZURE': - if (_.includes(OPTIMAL_MODES, pool_info.mode)) { - cloud_pool_stats.cloud_pool_target.azure += 1; - } else { + cloud_pool_stats.cloud_pool_target.azure += 1; + if (!_.includes(OPTIMAL_MODES, pool_info.mode)) { cloud_pool_stats.unhealthy_cloud_pool_target.azure_unhealthy += 1; } break; case 'GOOGLE': - if (_.includes(OPTIMAL_MODES, pool_info.mode)) { - cloud_pool_stats.cloud_pool_target.gcp += 1; - } else { + cloud_pool_stats.cloud_pool_target.gcp += 1; + if (!_.includes(OPTIMAL_MODES, pool_info.mode)) { cloud_pool_stats.unhealthy_cloud_pool_target.gcp_unhealthy += 1; } break; case 'S3_COMPATIBLE': - if (_.includes(OPTIMAL_MODES, pool_info.mode)) { - cloud_pool_stats.cloud_pool_target.s3_comp += 1; - } else { + cloud_pool_stats.cloud_pool_target.s3_comp += 1; + if (!_.includes(OPTIMAL_MODES, pool_info.mode)) { cloud_pool_stats.unhealthy_cloud_pool_target.s3_comp_unhealthy += 1; } if (pool.cloud_pool_info.auth_method === 'AWS_V2') { @@ -479,9 +584,8 @@ async function get_cloud_pool_stats(req) { } break; default: - if (_.includes(OPTIMAL_MODES, pool_info.mode)) { - cloud_pool_stats.cloud_pool_target.other += 1; - } else { + cloud_pool_stats.cloud_pool_target.other += 1; + if (!_.includes(OPTIMAL_MODES, pool_info.mode)) { cloud_pool_stats.unhealthy_cloud_pool_target.other_unhealthy += 1; } break; @@ -587,6 +691,7 @@ async function get_partial_stats(req) { const stats_payload = { systems_stats: null, cloud_pool_stats: null, + accounts_stats: null, }; dbg.log2('SYSTEM_SERVER_STATS_AGGREGATOR:', 'BEGIN'); @@ -606,6 +711,13 @@ async function get_partial_stats(req) { dbg.warn('Error in collecting cloud pool stats, skipping', error.stack, error); } + try { + dbg.log2('SYSTEM_SERVER_STATS_AGGREGATOR:', ' Collecting Partial Account I/O Stats'); + stats_payload.accounts_stats = await get_partial_accounts_stats(req); + } catch (error) { + dbg.warn('Error in collecting partial account i/o stats, skipping', error.stack, error); + } + partial_cycle_parse_prometheus_metrics(stats_payload); dbg.log2('SYSTEM_SERVER_STATS_AGGREGATOR:', 'END'); @@ -616,13 +728,10 @@ async function get_partial_stats(req) { * Prometheus Metrics Parsing, POC grade */ function partial_cycle_parse_prometheus_metrics(payload) { - const { cloud_pool_stats, systems_stats } = payload; + const { cloud_pool_stats, systems_stats, accounts_stats } = payload; // TODO: Support multiple systems - const { buckets_stats, free_space, total_space, name } = systems_stats.systems[0]; + const { buckets_stats, capacity, reduction_ratio, savings, name, usage_by_bucket_class, usage_by_project } = systems_stats.systems[0]; const { buckets, objects_in_buckets, unhealthy_buckets, bucket_claims, objects_in_bucket_claims } = buckets_stats; - const free_bytes = size_utils.bigint_to_bytes(free_space); - const total_bytes = size_utils.bigint_to_bytes(total_space); - const capacity = 100 - Math.floor((free_bytes / total_bytes) * 100); prom_report.instance().set_cloud_types(cloud_pool_stats); prom_report.instance().set_unhealthy_cloud_types(cloud_pool_stats); @@ -633,6 +742,11 @@ function partial_cycle_parse_prometheus_metrics(payload) { prom_report.instance().set_num_buckets_claims(bucket_claims); prom_report.instance().set_num_objects_buckets_claims(objects_in_bucket_claims); prom_report.instance().set_system_capacity(capacity); + prom_report.instance().set_reduction_ratio(reduction_ratio); + prom_report.instance().set_object_savings(savings); + prom_report.instance().set_bucket_class_capacity_usage(usage_by_bucket_class); + prom_report.instance().set_projects_capacity_usage(usage_by_project); + prom_report.instance().set_accounts_io_usage(accounts_stats); } /* @@ -979,6 +1093,7 @@ exports.get_cloud_pool_stats = get_cloud_pool_stats; exports.get_tier_stats = get_tier_stats; exports.get_all_stats = get_all_stats; exports.get_partial_systems_stats = get_partial_systems_stats; +exports.get_partial_accounts_stats = get_partial_accounts_stats; exports.get_partial_stats = get_partial_stats; exports.get_bucket_sizes_stats = get_bucket_sizes_stats; exports.get_object_usage_stats = get_object_usage_stats;