reading ACTIVATIONS_SCALE_FACTOR from rt_info

openvinotoolkit · Nov 12, 2024 · 5820e2b · 5820e2b
1 parent 4dd2238
commit 5820e2b
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 0 deletions.
diff --git a/src/inference/src/dev/core_impl.cpp b/src/inference/src/dev/core_impl.cpp
@@ -736,6 +736,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
     ov::AnyMap config_with_batch = config;
     // if auto-batching is applicable, the below function will patch the device name and config accordingly:
     auto model = apply_auto_batching(model_, deviceName, config_with_batch);
+    apply_rt_info(model_, config_with_batch);
 
     auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(device_name));
     auto plugin = get_plugin(parsed._deviceName);
@@ -769,6 +770,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
     ov::AnyMap config_with_batch = config;
     // if auto-batching is applicable, the below function will patch the device name and config accordingly:
     auto model = apply_auto_batching(model_, deviceName, config_with_batch);
+    apply_rt_info(model_, config_with_batch);
 
     auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName));
     auto plugin = get_plugin(parsed._deviceName);
@@ -1098,6 +1100,17 @@ std::shared_ptr<const ov::Model> ov::CoreImpl::apply_auto_batching(const std::sh
     return ov::details::apply_batch_affinity(model, deviceNameWithoutBatch);
 }
 
+void ov::CoreImpl::apply_rt_info(const std::shared_ptr<const ov::Model>& model,
+                                 ov::AnyMap& config) const {
+    if (model->has_rt_info({"runtime_options", "ACTIVATIONS_SCALE_FACTOR"})) {
+        if (config.find("ACTIVATIONS_SCALE_FACTOR") == config.end()) {
+            const auto activations_scale_factor =
+                model->get_rt_info<std::float_t>({"runtime_options", "ACTIVATIONS_SCALE_FACTOR"});
+            config.insert(ov::hint::activations_scale_factor(activations_scale_factor));
+        }
+    }
+}
+
 void ov::CoreImpl::set_property(const std::string& device_name, const AnyMap& properties) {
     OPENVINO_ASSERT(device_name.find("HETERO:") != 0,
                     "set_property is supported only for HETERO itself (without devices). "

diff --git a/src/inference/src/dev/core_impl.hpp b/src/inference/src/dev/core_impl.hpp
@@ -200,6 +200,9 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this<ov::ICore
                                                          std::string& deviceName,
                                                          ov::AnyMap& config) const;
 
+    void apply_rt_info(const std::shared_ptr<const ov::Model>& model,
+                       ov::AnyMap& config) const;
+
     /*
      * @brief Register plugins according to the build configuration
      */