forked from facebookincubator/velox
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add S3 metrics collection and reporting.
- Loading branch information
Showing
10 changed files
with
348 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #pragma once | ||
|
|
||
| namespace facebook::velox::filesystems { | ||
|
|
||
| /// Metric names for S3 FileSystem. | ||
| /// These metrics are used for monitoring and reporting various S3 operations. | ||
| constexpr auto kMetricS3ActiveConnections = | ||
| "presto_hive_s3_presto_s3_file_system_active_connections_total_count"; | ||
| constexpr auto kMetricS3StartedUploads = | ||
| "presto_hive_s3_presto_s3_file_system_started_uploads_one_minute_count"; | ||
| constexpr auto kMetricS3FailedUploads = | ||
| "presto_hive_s3_presto_s3_file_system_failed_uploads_one_minute_count"; | ||
| constexpr auto kMetricS3SuccessfulUploads = | ||
| "presto_hive_s3_presto_s3_file_system_successful_uploads_one_minute_count"; | ||
| constexpr auto kMetricS3MetadataCalls = | ||
| "presto_hive_s3_presto_s3_file_system_metadata_calls_one_minute_count"; | ||
| constexpr auto kMetricS3ListStatusCalls = | ||
| "presto_hive_s3_presto_s3_file_system_list_status_calls_one_minute_count"; | ||
| constexpr auto kMetricS3ListLocatedStatusCalls = | ||
| "presto_hive_s3_presto_s3_file_system_list_located_status_calls_one_minute_count"; | ||
| constexpr auto kMetricS3ListObjectsCalls = | ||
| "presto_hive_s3_presto_s3_file_system_list_objects_calls_one_minute_count"; | ||
| constexpr auto kMetricS3OtherReadErrors = | ||
| "presto_hive_s3_presto_s3_file_system_other_read_errors_one_minute_count"; | ||
| constexpr auto kMetricS3AwsAbortedExceptions = | ||
| "presto_hive_s3_presto_s3_file_system_aws_aborted_exceptions_one_minute_count"; | ||
| constexpr auto kMetricS3SocketExceptions = | ||
| "presto_hive_s3_presto_s3_file_system_socket_exceptions_one_minute_count"; | ||
| constexpr auto kMetricS3GetObjectErrors = | ||
| "presto_hive_s3_presto_s3_file_system_get_object_errors_one_minute_count"; | ||
| constexpr auto kMetricS3GetMetadataErrors = | ||
| "presto_hive_s3_presto_s3_file_system_get_metadata_errors_one_minute_count"; | ||
| constexpr auto kMetricS3GetObjectRetries = | ||
| "presto_hive_s3_presto_s3_file_system_get_object_retries_one_minute_count"; | ||
| constexpr auto kMetricS3GetMetadataRetries = | ||
| "presto_hive_s3_presto_s3_file_system_get_metadata_retries_one_minute_count"; | ||
| constexpr auto kMetricS3ReadRetries = | ||
| "presto_hive_s3_presto_s3_file_system_read_retries_one_minute_count"; | ||
|
|
||
| } // namespace facebook::velox::filesystems |
77 changes: 77 additions & 0 deletions
77
velox/connectors/hive/storage_adapters/s3fs/S3MetricsAggregator.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include "velox/connectors/hive/storage_adapters/s3fs/S3MetricsAggregator.h" | ||
| #include <glog/logging.h> | ||
| #include <folly/Singleton.h> | ||
|
|
||
| namespace facebook::velox::filesystems { | ||
|
|
||
| namespace { | ||
| folly::Singleton<S3MetricsAggregator> s3MetricsAggregatorSingleton; | ||
| } // namespace | ||
|
|
||
| std::shared_ptr<S3MetricsAggregator> S3MetricsAggregator::getInstance() { | ||
| return s3MetricsAggregatorSingleton.try_get(); | ||
| } | ||
|
|
||
| S3MetricsAggregator::S3MetricsAggregator() { | ||
| std::lock_guard<std::mutex> guard(mutex_); | ||
| std::vector<std::string> metricNames = { | ||
| "presto_hive_s3_presto_s3_file_system_active_connections_total_count", | ||
| "presto_hive_s3_presto_s3_file_system_started_uploads_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_failed_uploads_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_successful_uploads_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_metadata_calls_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_list_status_calls_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_list_located_status_calls_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_list_objects_calls_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_other_read_errors_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_aws_aborted_exceptions_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_socket_exceptions_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_get_object_errors_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_get_metadata_errors_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_get_object_retries_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_get_metadata_retries_one_minute_count", | ||
| "presto_hive_s3_presto_s3_file_system_read_retries_one_minute_count"}; | ||
|
|
||
| for (const auto& name : metricNames) { | ||
| metrics_[name] = 0; | ||
| } | ||
| } | ||
|
|
||
| // Increment the specified metric | ||
| void S3MetricsAggregator::incrementMetric(const std::string& metricName) { | ||
| std::lock_guard<std::mutex> guard(mutex_); | ||
| metrics_[metricName]++; | ||
| LOG(INFO) << "Incremented metric " << metricName << " to " | ||
| << metrics_[metricName]; | ||
| } | ||
|
|
||
| // Retrieve the current value of the specified metric | ||
| uint64_t S3MetricsAggregator::getMetric(const std::string& metricName) { | ||
| std::lock_guard<std::mutex> guard(mutex_); | ||
| return metrics_[metricName]; | ||
| } | ||
|
|
||
| // Reset the specified metric | ||
| void S3MetricsAggregator::resetMetric(const std::string& metricName) { | ||
| std::lock_guard<std::mutex> guard(mutex_); | ||
| metrics_[metricName] = 0; | ||
| LOG(INFO) << "Reset metric " << metricName << " to 0"; | ||
| } | ||
|
|
||
| } // namespace facebook::velox::filesystems |
43 changes: 43 additions & 0 deletions
43
velox/connectors/hive/storage_adapters/s3fs/S3MetricsAggregator.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #ifndef VELOX_CONNECTORS_HIVE_STORAGE_ADAPTERS_S3FS_S3METRICSAGGREGATOR_H_ | ||
| #define VELOX_CONNECTORS_HIVE_STORAGE_ADAPTERS_S3FS_S3METRICSAGGREGATOR_H_ | ||
|
|
||
| #include <folly/Singleton.h> | ||
| #include <mutex> | ||
| #include <unordered_map> | ||
|
|
||
| namespace facebook::velox::filesystems { | ||
|
|
||
| class S3MetricsAggregator { | ||
| public: | ||
| static std::shared_ptr<S3MetricsAggregator> getInstance(); | ||
|
|
||
| void incrementMetric(const std::string& metricName); | ||
| uint64_t getMetric(const std::string& metricName); | ||
| void resetMetric(const std::string& metricName); | ||
|
|
||
| private: | ||
| S3MetricsAggregator(); | ||
| std::unordered_map<std::string, uint64_t> metrics_; | ||
| std::mutex mutex_; | ||
| friend class folly::Singleton<S3MetricsAggregator>; | ||
| }; | ||
|
|
||
| } // namespace facebook::velox::filesystems | ||
|
|
||
| #endif // VELOX_CONNECTORS_HIVE_STORAGE_ADAPTERS_S3FS_S3METRICSAGGREGATOR_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.