From 0c8fc87ca185f1b2ebbd142e58f616688d9748c7 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 18 Nov 2024 17:38:47 +0100 Subject: [PATCH 01/12] add supprt for model outputs without names --- src/modelinstance.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 7497be90bd..fdd1d751be 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -305,9 +305,13 @@ static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_pt } OV_LOGGER("ov::Model: {}, model->outputs()", reinterpret_cast(model.get())); - for (const ov::Output& output : model->outputs()) { + for (ov::Output& output : model->outputs()) { try { OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); + if (output.get_names().size() == 0){ + std::unordered_set dummy_name{"out_"+std::to_string(output.get_index())}; + output.add_names(dummy_name); + } std::string name = output.get_any_name(); std::string mappedName = config.getMappingOutputByKey(name).empty() ? name : config.getMappingOutputByKey(name); if (config.getLayouts().count(mappedName) > 0) { From d62b6d9914cea7bbf923e3d6d2adf9688dfe36d8 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 18 Nov 2024 17:56:00 +0100 Subject: [PATCH 02/12] style --- src/modelinstance.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index fdd1d751be..4dc601a96b 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -308,8 +308,8 @@ static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_pt for (ov::Output& output : model->outputs()) { try { OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); - if (output.get_names().size() == 0){ - std::unordered_set dummy_name{"out_"+std::to_string(output.get_index())}; + if (output.get_names().size() == 0) { + std::unordered_set dummy_name{"out_" + std::to_string(output.get_index())}; output.add_names(dummy_name); } std::string name = output.get_any_name(); From 69c201c5a81047d1298c46527c1b56c73d4b9658 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 18 Nov 2024 22:36:12 +0100 Subject: [PATCH 03/12] style --- src/modelinstance.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 4dc601a96b..57a0fead91 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include // TODO windows From ceaf2021e003b3e0b16e15958d504fa994e4366a Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 18 Nov 2024 22:43:28 +0100 Subject: [PATCH 04/12] style --- src/modelinstance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 57a0fead91..c3f0485e85 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include #include // TODO windows From 90f5ca8af6733040b01ff150fa9df89c2c991e1a Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 16 Dec 2024 13:56:12 +0100 Subject: [PATCH 05/12] test dummy model --- src/test/no_name_output/1/model.bin | 0 src/test/no_name_output/1/model.xml | 86 +++++++++++++++++++++++++++++ tests/models/no_name_output.py | 47 ++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 src/test/no_name_output/1/model.bin create mode 100644 src/test/no_name_output/1/model.xml create mode 100644 tests/models/no_name_output.py diff --git a/src/test/no_name_output/1/model.bin b/src/test/no_name_output/1/model.bin new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/test/no_name_output/1/model.xml b/src/test/no_name_output/1/model.xml new file mode 100644 index 0000000000..d7c0a81b8b --- /dev/null +++ b/src/test/no_name_output/1/model.xml @@ -0,0 +1,86 @@ + + + + + + + + 1 + 10 + + + + + + + + 1 + 10 + + + + + + + + 1 + 10 + + + 1 + 10 + + + + + 1 + 10 + + + + + + + + 1 + 10 + + + 1 + 10 + + + + + 1 + 10 + + + + + + + 1 + 10 + + + + + + + 1 + 10 + + + + + + + + + + + + + + diff --git a/tests/models/no_name_output.py b/tests/models/no_name_output.py new file mode 100644 index 0000000000..9470feef21 --- /dev/null +++ b/tests/models/no_name_output.py @@ -0,0 +1,47 @@ +import openvino.runtime as ov +import numpy as np +import os +batch_dim = [] +shape = [1, 10] +dtype = np.int32 +model_name = "no_name_output" +model_version_dir = model_name +print(batch_dim + shape) +in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT1") +in1 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT2") +#tmp1 = ov.opset1.add(in0, start) +#tmp2 = ov.opset1.multiply(end, corrid) +#tmp = ov.opset1.add(tmp1, tmp2) +op0 = ov.opset1.multiply(in1, in0, name="MULTIPLY") +op1 = ov.opset1.add(in1, in0, name="ADD") + +# op0 = (input+start+(end*corrid))*ready + +model = ov.Model([op0, op1], [in0, in1], model_name) + +for idx, inp in enumerate(model.inputs): + print(f"Input {idx}: {inp.get_names()} {inp.get_shape()}") + +for idx, out in enumerate(model.outputs): + print(f"Output {idx}: {out.get_names()} {out.get_shape()}") + +try: + os.makedirs(model_version_dir) +except OSError as ex: + pass # ignore existing dir + +ov.serialize(model, model_version_dir + "/model.xml", model_version_dir + "/model.bin") + +ov_model = ov.Core().read_model(model_version_dir + "/model.xml") +compiled_model = ov.Core().compile_model(model, "CPU") + +input_data = np.ones((1, 10),dtype=np.int32)*20 +results = compiled_model({"INPUT1": input_data, "INPUT2": input_data}) + +print(input_data) +print(results) + +for idx, out in enumerate(ov_model.outputs): + out.get_tensor().set_names({f"out_{idx}"}) + +ov.save_model(ov_model, "model_with_names.xml") # it saves model with names \ No newline at end of file From 5fed196b1b4036c9153ee32f9ac889468de9aaa0 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Mon, 16 Dec 2024 13:56:55 +0100 Subject: [PATCH 06/12] fix --- src/test/no_name_output/1/model.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/no_name_output/1/model.xml b/src/test/no_name_output/1/model.xml index d7c0a81b8b..eedba5be53 100644 --- a/src/test/no_name_output/1/model.xml +++ b/src/test/no_name_output/1/model.xml @@ -1,5 +1,5 @@ - + From 19c89280cfebb11b3b4e4dbc1bbad87fd69f2dca Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Tue, 17 Dec 2024 12:57:50 +0100 Subject: [PATCH 07/12] unit tests for no_name model --- src/test/modelinstance_test.cpp | 12 ++++++++++++ src/test/test_utils.hpp | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/test/modelinstance_test.cpp b/src/test/modelinstance_test.cpp index 0a8db6edf3..54d826be53 100644 --- a/src/test/modelinstance_test.cpp +++ b/src/test/modelinstance_test.cpp @@ -112,6 +112,18 @@ TEST_F(TestUnloadModel, CanUnloadModelNotHoldingModelInstanceAtPredictPath) { EXPECT_TRUE(modelInstance.canUnloadInstance()); } +TEST_F(TestUnloadModel, NoNameOutput) { + ovms::ModelInstance modelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, *ieCore); + ASSERT_EQ(modelInstance.loadModel(NO_NAME_MODEL_CONFIG), ovms::StatusCode::OK); + ASSERT_EQ(ovms::ModelVersionState::AVAILABLE, modelInstance.getStatus().getState()); + EXPECT_EQ(modelInstance.getInputsInfo().count("INPUT1"), 1); + EXPECT_EQ(modelInstance.getInputsInfo().count("INPUT2"), 1); + EXPECT_EQ(modelInstance.getOutputsInfo().count("out_0"), 1); + EXPECT_EQ(modelInstance.getOutputsInfo().count("out_1"), 1); + modelInstance.retireModel(); + EXPECT_EQ(ovms::ModelVersionState::END, modelInstance.getStatus().getState()); +} + TEST_F(TestUnloadModel, UnloadWaitsUntilMetadataResponseIsBuilt) { static std::thread thread; static std::shared_ptr instance; diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 67cedafc13..9af6c88436 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -70,6 +70,7 @@ const std::string passthrough_string_model_location = std::filesystem::current_p const std::string dummy_saved_model_location = std::filesystem::current_path().u8string() + "/src/test/dummy_saved_model"; const std::string dummy_tflite_location = std::filesystem::current_path().u8string() + "/src/test/dummy_tflite"; const std::string scalar_model_location = std::filesystem::current_path().u8string() + "/src/test/scalar"; +const std::string no_name_output_model_location = std::filesystem::current_path().u8string() + "/src/test/no_name_output"; const ovms::ModelConfig DUMMY_MODEL_CONFIG{ "dummy", @@ -206,6 +207,21 @@ const ovms::ModelConfig SCALAR_MODEL_CONFIG{ scalar_model_location, // local path }; +const ovms::ModelConfig NO_NAME_MODEL_CONFIG{ + "no_name_output", + no_name_output_model_location, // base path + "CPU", // target device + "1", // batchsize + 1, // NIREQ + false, // is stateful + true, // idle sequence cleanup enabled + false, // low latency transformation enabled + 500, // stateful sequence max number + "", // cache directory + 1, // model_version unused since version are read from path + no_name_output_model_location, // local path +}; + constexpr const char* DUMMY_MODEL_INPUT_NAME = "b"; constexpr const char* DUMMY_MODEL_OUTPUT_NAME = "a"; constexpr const int DUMMY_MODEL_INPUT_SIZE = 10; From 5a84eface24a0c5e59ef556c760928ef90782bc5 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Tue, 17 Dec 2024 15:25:15 +0100 Subject: [PATCH 08/12] style --- src/modelinstance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 837243328a..51b523f078 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -309,7 +309,7 @@ static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_pt size_t outputIndex = 0; for (ov::Output& output : model->outputs()) { try { - OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); + OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); if (output.get_names().size() == 0) { std::unordered_set dummy_name{"OUT_" + std::to_string(outputIndex)}; output.add_names(dummy_name); From 2dae97cdf70758000057f88e03fb18eacd80dbaa Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sun, 12 Jan 2025 01:51:12 +0100 Subject: [PATCH 09/12] review changes --- src/modelinstance.cpp | 29 +++++++++++++++++++++++------ src/test/modelinstance_test.cpp | 8 ++++---- src/test/no_name_output/1/model.xml | 8 ++++---- tests/models/no_name_output.py | 14 +++++++------- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 51b523f078..63a5f21b44 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -306,14 +306,9 @@ static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_pt } OV_LOGGER("ov::Model: {}, model->outputs()", reinterpret_cast(model.get())); - size_t outputIndex = 0; for (ov::Output& output : model->outputs()) { try { OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); - if (output.get_names().size() == 0) { - std::unordered_set dummy_name{"OUT_" + std::to_string(outputIndex)}; - output.add_names(dummy_name); - } std::string name = output.get_any_name(); std::string mappedName = config.getMappingOutputByKey(name).empty() ? name : config.getMappingOutputByKey(name); if (config.getLayouts().count(mappedName) > 0) { @@ -361,7 +356,6 @@ static Status applyLayoutConfiguration(const ModelConfig& config, std::shared_pt modelVersion); return StatusCode::UNKNOWN_ERROR; } - outputIndex++; } try { @@ -420,6 +414,29 @@ Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLa SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error during configuration validation against model"); return status; } + // add output names if not present in the model + size_t outputIndex = 0; + for (ov::Output& output : model->outputs()) { + try { + OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); + if (output.get_names().size() == 0) { + std::unordered_set dummy_name{"out_" + std::to_string(outputIndex)}; + output.add_names(dummy_name); + } + } catch (const ov::Exception& e) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to set the missing name in output for model:{}; version:{}; Error:{}", + modelName, + modelVersion, + e.what()); + return StatusCode::UNKNOWN_ERROR; + } catch (...) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to set the missing name in output for model:{}; version:{};", + getName(), + getVersion()); + return StatusCode::UNKNOWN_ERROR; + } + outputIndex++; + } if (needsToApplyLayoutConfiguration) { status = applyLayoutConfiguration(config, this->model, getName(), getVersion()); if (!status.ok()) { diff --git a/src/test/modelinstance_test.cpp b/src/test/modelinstance_test.cpp index 36518082c9..a6a8601d79 100644 --- a/src/test/modelinstance_test.cpp +++ b/src/test/modelinstance_test.cpp @@ -116,10 +116,10 @@ TEST_F(TestUnloadModel, NoNameOutput) { ovms::ModelInstance modelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, *ieCore); ASSERT_EQ(modelInstance.loadModel(NO_NAME_MODEL_CONFIG), ovms::StatusCode::OK); ASSERT_EQ(ovms::ModelVersionState::AVAILABLE, modelInstance.getStatus().getState()); - EXPECT_EQ(modelInstance.getInputsInfo().count("INPUT1"), 1); - EXPECT_EQ(modelInstance.getInputsInfo().count("INPUT2"), 1); - EXPECT_EQ(modelInstance.getOutputsInfo().count("OUT_0"), 1); - EXPECT_EQ(modelInstance.getOutputsInfo().count("OUT_1"), 1); + EXPECT_EQ(modelInstance.getInputsInfo().count("input1"), 1); + EXPECT_EQ(modelInstance.getInputsInfo().count("input2"), 1); + EXPECT_EQ(modelInstance.getOutputsInfo().count("out_0"), 1); + EXPECT_EQ(modelInstance.getOutputsInfo().count("out_1"), 1); modelInstance.retireModel(); EXPECT_EQ(ovms::ModelVersionState::END, modelInstance.getStatus().getState()); } diff --git a/src/test/no_name_output/1/model.xml b/src/test/no_name_output/1/model.xml index f9f3585906..3341cbf2e7 100644 --- a/src/test/no_name_output/1/model.xml +++ b/src/test/no_name_output/1/model.xml @@ -1,19 +1,19 @@ - + - + 1 10 - + - + 1 10 diff --git a/tests/models/no_name_output.py b/tests/models/no_name_output.py index 6b2877e29e..d0e7fea7d3 100644 --- a/tests/models/no_name_output.py +++ b/tests/models/no_name_output.py @@ -24,8 +24,8 @@ model_name = "no_name_output" model_version_dir = model_name print(batch_dim + shape) -in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT1") -in1 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT2") +in0 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="input1") +in1 = ov.opset1.parameter(shape=batch_dim + shape, dtype=dtype, name="input2") op0 = ov.opset1.multiply(in1, in0, name="MULTIPLY") op1 = ov.opset1.add(in1, in0, name="ADD") @@ -35,7 +35,8 @@ print(f"Input {idx}: {inp.get_names()} {inp.get_shape()} {inp.get_index()}") print(model.outputs) for idx, out in enumerate(model.outputs): - print(f"Output {idx}: {out.get_names()} {out.get_shape()} {out.get_index()} {out.get_any_name()}") + print(f"Output {idx}: {out.get_names()} {out.get_shape()} {out.get_index()} ") + assert len(out.get_names()) == 0, "number of output names should be 0" try: os.makedirs(model_version_dir) @@ -48,8 +49,7 @@ compiled_model = ov.Core().compile_model(model, "CPU") input_data = np.ones((1, 10),dtype=np.int8)*10 -results = compiled_model({"INPUT1": input_data, "INPUT2": input_data}) - -print(input_data) -print(results) +results = compiled_model({"input1": input_data, "input2": input_data}) +assert np.all(results[0] == 100), "for inputs np.ones((1, 10), the expected output is 100 in every element: 10*10" +assert np.all(results[1] == 20), "for inputs np.ones((1, 10), the expected output is 20 in every element: 10+10" From 698c871a3a1223035db9e4f41bb27539bc07bbbc Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sun, 12 Jan 2025 02:09:42 +0100 Subject: [PATCH 10/12] fix --- src/modelinstance.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index a935a017c2..9bcae25183 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -416,7 +416,7 @@ Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLa } // add output names if not present in the model size_t outputIndex = 0; - for (ov::Output& output : model->outputs()) { + for (ov::Output& output : this->model->outputs()) { try { OV_LOGGER("ov::Output output: {}, output.get_any_name()", reinterpret_cast(&output)); if (output.get_names().size() == 0) { @@ -425,8 +425,8 @@ Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLa } } catch (const ov::Exception& e) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to set the missing name in output for model:{}; version:{}; Error:{}", - modelName, - modelVersion, + getName(), + getVersion() e.what()); return StatusCode::UNKNOWN_ERROR; } catch (...) { From 4f02072608a932876bb3b484e0cb36d1b9adff94 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sun, 12 Jan 2025 22:32:48 +0100 Subject: [PATCH 11/12] fix --- src/modelinstance.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 9bcae25183..48c2cbbcfa 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -426,7 +426,7 @@ Status ModelInstance::loadTensors(const ModelConfig& config, bool needsToApplyLa } catch (const ov::Exception& e) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to set the missing name in output for model:{}; version:{}; Error:{}", getName(), - getVersion() + getVersion(), e.what()); return StatusCode::UNKNOWN_ERROR; } catch (...) { From 028a2e486400bf9e9db576fe6739adda3294cca8 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sun, 12 Jan 2025 23:37:26 +0100 Subject: [PATCH 12/12] header --- tests/models/no_name_output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/no_name_output.py b/tests/models/no_name_output.py index d0e7fea7d3..af255dfedd 100644 --- a/tests/models/no_name_output.py +++ b/tests/models/no_name_output.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2020 Intel Corporation +# Copyright (c) 2025 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.