Consolidate base config classes (#10694)

Summary:
There are a couple of base config classes in the current codebase, while serving similar purposes. It is often confusing. Consolidate these classes to make codebase cleaner.

Pull Request resolved: #10694

Reviewed By: xiaoxmeng

Differential Revision: D61028580

Pulled By: tanjialiang

fbshipit-source-id: 4a85b4e0a8cae0d39b7f689993021b3ccef61829

velox/common/CMakeLists.txt

@@ -14,6 +14,7 @@
 add_subdirectory(base)
 add_subdirectory(caching)
 add_subdirectory(compression)
+add_subdirectory(config)
 add_subdirectory(encode)
 add_subdirectory(file)
 add_subdirectory(hyperloglog)

velox/common/config/CMakeLists.txt

@@ -0,0 +1,24 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if (${VELOX_BUILD_TESTING})
+  add_subdirectory(tests)
+endif ()
+
+velox_add_library(velox_common_config Config.cpp)
+velox_link_libraries(
+  velox_common_config
+  PUBLIC velox_common_base
+         velox_exception
+  PRIVATE re2::re2)

velox/common/config/Config.cpp

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <re2/re2.h>
+
+#include "velox/common/config/Config.h"
+
+namespace facebook::velox::config {
+
+double toBytesPerCapacityUnit(CapacityUnit unit) {
+  switch (unit) {
+    case CapacityUnit::BYTE:
+      return 1;
+    case CapacityUnit::KILOBYTE:
+      return exp2(10);
+    case CapacityUnit::MEGABYTE:
+      return exp2(20);
+    case CapacityUnit::GIGABYTE:
+      return exp2(30);
+    case CapacityUnit::TERABYTE:
+      return exp2(40);
+    case CapacityUnit::PETABYTE:
+      return exp2(50);
+    default:
+      VELOX_USER_FAIL("Invalid capacity unit '{}'", (int)unit);
+  }
+}
+
+CapacityUnit valueOfCapacityUnit(const std::string& unitStr) {
+  std::stringstream ss;
+  for (const char c : unitStr) {
+    ss << static_cast<char>(std::tolower(c));
+  }
+  const auto lowerUnitStr = ss.str();
+  if (lowerUnitStr == "b") {
+    return CapacityUnit::BYTE;
+  }
+  if (lowerUnitStr == "kb") {
+    return CapacityUnit::KILOBYTE;
+  }
+  if (lowerUnitStr == "mb") {
+    return CapacityUnit::MEGABYTE;
+  }
+  if (lowerUnitStr == "gb") {
+    return CapacityUnit::GIGABYTE;
+  }
+  if (lowerUnitStr == "tb") {
+    return CapacityUnit::TERABYTE;
+  }
+  if (lowerUnitStr == "pb") {
+    return CapacityUnit::PETABYTE;
+  }
+  VELOX_USER_FAIL("Invalid capacity unit '{}'", unitStr);
+}
+
+uint64_t toCapacity(const std::string& from, CapacityUnit to) {
+  static const RE2 kPattern(R"(^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*$)");
+  double value;
+  std::string unit;
+  if (!RE2::FullMatch(from, kPattern, &value, &unit)) {
+    VELOX_USER_FAIL("Invalid capacity string '{}'", from);
+  }
+
+  return value *
+      (toBytesPerCapacityUnit(valueOfCapacityUnit(unit)) /
+       toBytesPerCapacityUnit(to));
+}
+
+std::chrono::duration<double> toDuration(const std::string& str) {
+  static const RE2 kPattern(R"(^\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]+)\s*)");
+
+  double value;
+  std::string unit;
+  if (!RE2::FullMatch(str, kPattern, &value, &unit)) {
+    VELOX_USER_FAIL("Invalid duration '{}'", str);
+  }
+  if (unit == "ns") {
+    return std::chrono::duration<double, std::nano>(value);
+  } else if (unit == "us") {
+    return std::chrono::duration<double, std::micro>(value);
+  } else if (unit == "ms") {
+    return std::chrono::duration<double, std::milli>(value);
+  } else if (unit == "s") {
+    return std::chrono::duration<double>(value);
+  } else if (unit == "m") {
+    return std::chrono::duration<double, std::ratio<60>>(value);
+  } else if (unit == "h") {
+    return std::chrono::duration<double, std::ratio<60 * 60>>(value);
+  } else if (unit == "d") {
+    return std::chrono::duration<double, std::ratio<60 * 60 * 24>>(value);
+  }
+  VELOX_USER_FAIL("Invalid duration '{}'", str);
+}
+
+ConfigBase& ConfigBase::set(const std::string& key, const std::string& val) {
+  VELOX_CHECK(mutable_, "Cannot set in immutable config");
+  std::unique_lock<std::shared_mutex> l(mutex_);
+  configs_[key] = val;
+  return *this;
+}
+
+ConfigBase& ConfigBase::reset() {
+  VELOX_CHECK(mutable_, "Cannot reset in immutable config");
+  std::unique_lock<std::shared_mutex> l(mutex_);
+  configs_.clear();
+  return *this;
+}
+
+bool ConfigBase::valueExists(const std::string& key) const {
+  std::shared_lock<std::shared_mutex> l(mutex_);
+  return configs_.find(key) != configs_.end();
+};
+
+const std::unordered_map<std::string, std::string>& ConfigBase::rawConfigs()
+    const {
+  VELOX_CHECK(
+      !mutable_,
+      "Mutable config cannot return unprotected reference to raw configs.");
+  return configs_;
+}
+
+std::unordered_map<std::string, std::string> ConfigBase::rawConfigsCopy()
+    const {
+  std::shared_lock<std::shared_mutex> l(mutex_);
+  return configs_;
+}
+
+folly::Optional<std::string> ConfigBase::get(const std::string& key) const {
+  folly::Optional<std::string> val;
+  std::shared_lock<std::shared_mutex> l(mutex_);
+  auto it = configs_.find(key);
+  if (it != configs_.end()) {
+    val = it->second;
+  }
+  return val;
+}
+} // namespace facebook::velox::config

velox/common/config/Config.h

@@ -23,79 +23,136 @@
 #include "folly/Conv.h"
 #include "velox/common/base/Exceptions.h"
 
-namespace facebook::velox::common {
-// The concrete config class would inherit the config base
-// and then just define all the entries.
-template <class ConcreteConfig>
+namespace facebook::velox::config {
+
+enum class CapacityUnit {
+  BYTE,
+  KILOBYTE,
+  MEGABYTE,
+  GIGABYTE,
+  TERABYTE,
+  PETABYTE
+};
+
+double toBytesPerCapacityUnit(CapacityUnit unit);
+
+CapacityUnit valueOfCapacityUnit(const std::string& unitStr);
+
+/// Convert capacity string with unit to the capacity number in the specified
+/// units
+uint64_t toCapacity(const std::string& from, CapacityUnit to);
+
+std::chrono::duration<double> toDuration(const std::string& str);
+
+/// The concrete config class should inherit the config base and define all the
+/// entries.
 class ConfigBase {
  public:
   template <typename T>
-  class Entry {
-   private:
+  struct Entry {
     Entry(
-        const std::string& key,
-        const T& val,
-        std::function<std::string(const T&)> toStr =
+        const std::string& _key,
+        const T& _val,
+        std::function<std::string(const T&)> _toStr =
             [](const T& val) { return folly::to<std::string>(val); },
-        std::function<T(const std::string&, const std::string&)> toT =
-            [](const std::string& key, const std::string& val) {
-              auto converted = folly::tryTo<T>(val);
+        std::function<T(const std::string&, const std::string&)> _toT =
+            [](const std::string& k, const std::string& v) {
+              auto converted = folly::tryTo<T>(v);
               VELOX_CHECK(
                   converted.hasValue(),
                   fmt::format(
                       "Invalid configuration for key '{}'. Value '{}' cannot be converted to type {}.",
-                      key,
-                      val,
+                      k,
+                      v,
                       folly::demangle(typeid(T))));
               return converted.value();
             })
-        : key_{key}, default_{val}, toStr_{toStr}, toT_{toT} {}
+        : key{_key}, defaultVal{_val}, toStr{_toStr}, toT{_toT} {}
 
-   public:
-    const std::string& configKey() const {
-      return key_;
-    }
-
-   private:
-    const std::string key_;
-    const T default_;
-    const std::function<std::string(const T&)> toStr_;
-    const std::function<T(const std::string&, const std::string&)> toT_;
-
-    friend ConfigBase;
-    friend ConcreteConfig;
+    const std::string key;
+    const T defaultVal;
+    const std::function<std::string(const T&)> toStr;
+    const std::function<T(const std::string&, const std::string&)> toT;
   };
 
+  ConfigBase(
+      std::unordered_map<std::string, std::string>&& configs,
+      bool _mutable = false)
+      : configs_(std::move(configs)), mutable_(_mutable) {}
+
   template <typename T>
   ConfigBase& set(const Entry<T>& entry, const T& val) {
-    configs_[entry.key_] = entry.toStr_(val);
+    VELOX_CHECK(mutable_, "Cannot set in immutable config");
+    std::unique_lock<std::shared_mutex> l(mutex_);
+    configs_[entry.key] = entry.toStr(val);
     return *this;
   }
 
+  ConfigBase& set(const std::string& key, const std::string& val);
+
   template <typename T>
   ConfigBase& unset(const Entry<T>& entry) {
-    configs_.erase(entry.key_);
+    VELOX_CHECK(mutable_, "Cannot unset in immutable config");
+    std::unique_lock<std::shared_mutex> l(mutex_);
+    configs_.erase(entry.key);
     return *this;
   }
 
-  ConfigBase& reset() {
-    configs_.clear();
-    return *this;
-  }
+  ConfigBase& reset();
 
   template <typename T>
   T get(const Entry<T>& entry) const {
-    auto iter = configs_.find(entry.key_);
-    return iter != configs_.end() ? entry.toT_(entry.key_, iter->second)
-                                  : entry.default_;
+    std::shared_lock<std::shared_mutex> l(mutex_);
+    auto iter = configs_.find(entry.key);
+    return iter != configs_.end() ? entry.toT(entry.key, iter->second)
+                                  : entry.defaultVal;
+  }
+
+  template <typename T>
+  folly::Optional<T> get(
+      const std::string& key,
+      std::function<T(std::string, std::string)> toT = [](auto /* unused */,
+                                                          auto value) {
+        return folly::to<T>(value);
+      }) const {
+    auto val = get(key);
+    if (val.hasValue()) {
+      return toT(key, val.value());
+    } else {
+      return folly::none;
+    }
   }
 
-  std::map<std::string, std::string> toSerdeParams() {
-    return std::map{configs_.cbegin(), configs_.cend()};
+  template <typename T>
+  T get(
+      const std::string& key,
+      const T& defaultValue,
+      std::function<T(std::string, std::string)> toT = [](auto /* unused */,
+                                                          auto value) {
+        return folly::to<T>(value);
+      }) const {
+    auto val = get(key);
+    if (val.hasValue()) {
+      return toT(key, val.value());
+    } else {
+      return defaultValue;
+    }
   }
 
+  bool valueExists(const std::string& key) const;
+
+  const std::unordered_map<std::string, std::string>& rawConfigs() const;
+
+  std::unordered_map<std::string, std::string> rawConfigsCopy() const;
+
  protected:
+  mutable std::shared_mutex mutex_;
   std::unordered_map<std::string, std::string> configs_;
+
+ private:
+  folly::Optional<std::string> get(const std::string& key) const;
+
+  const bool mutable_;
 };
 
-} // namespace facebook::velox::common
+} // namespace facebook::velox::config

velox/common/config/tests/CMakeLists.txt

@@ -0,0 +1,20 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_executable(velox_config_test ConfigTest.cpp)
+add_test(velox_config_test velox_config_test)
+target_link_libraries(
+  velox_config_test
+  PUBLIC Folly::folly
+  PRIVATE velox_common_config gtest gtest_main)