Skip to content

Commit

Permalink
Merge pull request #5585 from nimrod-becker/nimrod_backport_4_0
Browse files Browse the repository at this point in the history
Backport Prometheus Metrics phase2 into 4.0
  • Loading branch information
nimrod-becker authored Jul 1, 2019
2 parents 1a48828 + d584090 commit 1548373
Show file tree
Hide file tree
Showing 6 changed files with 288 additions and 59 deletions.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "noobaa-core",
"version": "4.0.1",
"version": "4.0.2",
"private": true,
"license": "SEE LICENSE IN LICENSE",
"description": "",
Expand Down
66 changes: 60 additions & 6 deletions src/api/stats_api.js
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ module.exports = {
}
},

get_partial_accounts_stats: {
method: 'GET',
reply: {
$ref: '#/definitions/partial_accounts_stats'
},
auth: {
system: 'admin'
}
},

get_nodes_stats: {
method: 'GET',
reply: {
Expand Down Expand Up @@ -483,14 +493,17 @@ module.exports = {

partial_stats: {
type: 'object',
required: ['systems_stats', 'cloud_pool_stats'],
required: ['systems_stats', 'cloud_pool_stats', 'accounts_stats'],
properties: {
systems_stats: {
$ref: '#/definitions/partial_systems_stats'
},
cloud_pool_stats: {
$ref: '#/definitions/cloud_pool_stats'
},
accounts_stats: {
$ref: '#/definitions/partial_accounts_stats'
},
}
},

Expand All @@ -502,20 +515,61 @@ module.exports = {
type: 'array',
items: {
type: 'object',
required: ['name', 'free_space', 'total_space', 'buckets_stats'],
required: ['name', 'capacity', 'reduction_ratio', 'savings', 'buckets_stats', 'usage_by_project', 'usage_by_bucket_class'],
properties: {
name: {
type: 'string'
},
free_space: {
$ref: 'common_api#/definitions/bigint'
capacity: {
type: 'number'
},
total_space: {
$ref: 'common_api#/definitions/bigint'
savings: {
type: 'number'
},
reduction_ratio: {
type: 'number'
},
buckets_stats: {
$ref: '#/definitions/partial_buckets_stats'
},
usage_by_project: {
type: 'object',
additionalProperties: true,
properties: {},
},
usage_by_bucket_class: {
type: 'object',
additionalProperties: true,
properties: {},
},
}
}
}
}
},

partial_accounts_stats: {
type: 'object',
required: ['accounts'],
properties: {
accounts: {
type: 'array',
items: {
type: 'object',
required: ['account', 'read_count', 'write_count', 'read_write_bytes'],
properties: {
account: {
type: 'string'
},
read_count: {
type: 'number'
},
write_count: {
type: 'number'
},
read_write_bytes: {
type: 'number'
},
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/deploy/NVA_build/noobaa_core.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ spec:
tcpSocket:
port: 6001
timeoutSeconds: 5
image: noobaa/noobaa-core:4.0.1
image: noobaa/noobaa-core:4.0.2
imagePullPolicy: IfNotPresent
resources:
# https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
Expand Down
65 changes: 65 additions & 0 deletions src/server/analytic_services/prometheus_reporting.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,30 @@ const METRIC_RECORDS = Object.freeze([{
help: 'Cloud Resource Types in the System',
labelNames: ['type', 'count']
}
}, {
metric_type: 'Gauge',
metric_variable: 'projects_capacity_usage',
configuration: {
name: get_metric_name('projects_capacity_usage'),
help: 'Projects Capacity Usage',
labelNames: ['project', 'count']
}
}, {
metric_type: 'Gauge',
metric_variable: 'accounts_io_usage',
configuration: {
name: get_metric_name('accounts_io_usage'),
help: 'Accounts I/O Usage',
labelNames: ['account', 'read_count', 'write_count']
}
}, {
metric_type: 'Gauge',
metric_variable: 'bucket_class_capacity_usage',
configuration: {
name: get_metric_name('bucket_class_capacity_usage'),
help: 'Bucket Class Capacity Usage',
labelNames: ['bucket_class', 'count']
}
}, {
metric_type: 'Gauge',
metric_variable: 'unhealthy_cloud_types',
Expand Down Expand Up @@ -105,6 +129,22 @@ const METRIC_RECORDS = Object.freeze([{
help: 'Objects On Object Bucket Claims',
},
generate_default_set: true,
}, {
metric_type: 'Gauge',
metric_variable: 'reduction_ratio',
configuration: {
name: get_metric_name('reduction_ratio'),
help: 'Object Efficiency Ratio',
},
generate_default_set: true,
}, {
metric_type: 'Gauge',
metric_variable: 'object_savings',
configuration: {
name: get_metric_name('object_savings'),
help: 'Object Savings',
},
generate_default_set: true,
}]);


Expand Down Expand Up @@ -173,6 +213,31 @@ class PrometheusReporting {
this._metrics.unhealthy_cloud_types.set({ type: 'S3_Compatible' }, types.unhealthy_cloud_pool_target.s3_comp_unhealthy);
}

set_bucket_class_capacity_usage(usage_info) {
if (!this.enabled()) return;
this._metrics.bucket_class_capacity_usage.reset();
for (let [key, value] of Object.entries(usage_info)) {
this._metrics.bucket_class_capacity_usage.set({ bucket_class: key }, value);
}
}

set_projects_capacity_usage(usage_info) {
if (!this.enabled()) return;
this._metrics.projects_capacity_usage.reset();
for (let [key, value] of Object.entries(usage_info)) {
this._metrics.projects_capacity_usage.set({ project: key }, value);
}
}

set_accounts_io_usage(accounts_info) {
if (!this.enabled()) return;
this._metrics.accounts_io_usage.reset();
accounts_info.accounts.forEach(account_info => {
const { account, read_count, write_count, read_write_bytes } = account_info;
this._metrics.accounts_io_usage.set({ account, read_count, write_count }, read_write_bytes);
});
}

set_object_sizes(sizes) {
if (!this.enabled()) return;
for (const bin of sizes) {
Expand Down
35 changes: 15 additions & 20 deletions src/server/system_services/account_server.js
Original file line number Diff line number Diff line change
Expand Up @@ -706,31 +706,26 @@ function check_external_connection(req) {
return P.resolve()
.then(() => {
switch (endpoint_type) {
case 'AZURE':
{
return check_azure_connection(params);
}
case 'AZURE': {
return check_azure_connection(params);
}

case 'AWS':
case 'S3_COMPATIBLE':
case 'FLASHBLADE':
{
return check_aws_connection(params);
}
case 'FLASHBLADE': {
return check_aws_connection(params);
}

case 'NET_STORAGE':
{
return check_net_storage_connection(params);
}
case 'GOOGLE':
{
return check_google_connection(params);
}
case 'NET_STORAGE': {
return check_net_storage_connection(params);
}
case 'GOOGLE': {
return check_google_connection(params);
}

default:
{
throw new Error('Unknown endpoint type');
}
default: {
throw new Error('Unknown endpoint type');
}
}
});
}
Expand Down
Loading

0 comments on commit 1548373

Please sign in to comment.