Ovep mem optimization

include/onnxruntime/core/graph/graph.h

@@ -571,6 +571,13 @@ class Node {
             gsl::span<NodeArg* const> output_args,
             const NodeAttributes* attributes,
             std::string_view domain);
+  void Init(std::string_view name,
+            std::string_view op_type,
+            std::string_view description,
+            gsl::span<NodeArg* const> input_args,
+            gsl::span<NodeArg* const> output_args,
+            NodeAttributes&& attributes,
+            std::string_view domain);
 #endif
 
 #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@@ -952,6 +959,13 @@ class Graph {  // NOLINT(clang-analyzer-optin.performance.Padding): preserve exi
                 const NodeAttributes* attributes = nullptr,
                 const std::string& domain = kOnnxDomain);
 
+  Node& AddNode(const std::string& name,
+                const std::string& op_type,
+                const std::string& description,
+                gsl::span<NodeArg* const> input_args,
+                gsl::span<NodeArg* const> output_args,
+                NodeAttributes&& attributes,
+                const std::string& domain = kOnnxDomain);
   Node& AddNode(const std::string& name,
                 const std::string& op_type,
                 const std::string& description,

onnxruntime/core/graph/graph.cc

@@ -915,6 +915,42 @@ void Node::Init(std::string_view name,
     }
   }
 }
+void Node::Init(std::string_view name,
+                std::string_view op_type,
+                std::string_view description,
+                gsl::span<NodeArg* const> input_args,
+                gsl::span<NodeArg* const> output_args,
+                NodeAttributes&& attributes,
+                std::string_view domain) {
+  name_ = name;
+  op_type_ = op_type;
+  description_ = description;
+  definitions_.input_defs.assign(input_args.begin(), input_args.end());
+  definitions_.output_defs.assign(output_args.begin(), output_args.end());
+  domain_ = domain;
+  can_be_saved_ = true;
+  priority_ = 0;
+  if (kOnnxDomainAlias == domain_) {
+    domain_ = kOnnxDomain;
+  }
+
+  // Set each arg count as 1 by default.
+  // It could be adjusted when resolving the node with its operator
+  // information.
+  definitions_.input_arg_count.assign(input_args.size(), 1);
+
+  attributes_ = std::move(attributes);
+
+  for (auto& name_to_attr : attributes_) {
+    if (utils::HasGraph(name_to_attr.second)) {
+#if !defined(ORT_MINIMAL_BUILD)
+      CreateSubgraph(name_to_attr.first);
+#else
+      ORT_THROW("Creating node with a subgraph via AddNode is not supported in this build.");
+#endif
+    }
+  }
+}
 #endif  // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
 
 #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@@ -3923,6 +3959,35 @@ Node& Graph::AddNode(const std::string& name,
   return *node;
 }
 
+Node& Graph::AddNode(const std::string& name,
+                     const std::string& op_type,
+                     const std::string& description,
+                     gsl::span<NodeArg* const> input_args,
+                     gsl::span<NodeArg* const> output_args,
+                     NodeAttributes&& attributes,
+                     const std::string& domain) {
+  InlinedVector<NodeArg*> inputs;
+  InlinedVector<NodeArg*> outputs;
+  inputs.resize(input_args.size());
+  outputs.resize(output_args.size());
+  int i = 0;
+  for (auto input_arg : input_args) {
+    inputs[i++] = &GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto());
+  }
+  i = 0;
+  for (auto output_arg : output_args) {
+    outputs[i++] = &GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto());
+  }
+
+  const gsl::not_null<Node*> node = AllocateNode();
+  node->Init(name, op_type, description, inputs, outputs, std::move(attributes), domain);
+  if (0 != op_type.compare(kNoOp)) {
+    GraphProtoSyncNeeded(true);
+  }
+
+  return *node;
+}
+
 bool Graph::RemoveNode(NodeIndex p_index) {
   auto node = GetNode(p_index);
   if (nullptr == node) {

onnxruntime/core/providers/openvino/backend_manager.cc

@@ -155,32 +155,28 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
   auto compiled_model = concrete_backend_->GetOVCompiledModel();
   std::string graph_name = "";
   // Epctx file path from SO is mapped to cache_dir variable for OVEP for readability
-  if (global_context_.cache_dir != "") {
+  if (!global_context_.cache_dir.empty()) {
     graph_name = global_context_.cache_dir;
   } else {
     graph_name = global_context_.onnx_model_path_name;
     // Remove extension so we can append suffix to form the complete name of output graph
-    graph_name = [&]() {
-      size_t dot = graph_name.find_last_of(".");
-      if (dot == std::string::npos) return graph_name;
-      return graph_name.substr(0, dot);
-    }();
-    graph_name = graph_name + "_ctx.onnx";
+    size_t dot = global_context_.onnx_model_path_name.find_last_of(".");
+    graph_name = graph_name.substr(0, dot);
+    if (dot != std::string::npos) graph_name += "_ctx.onnx";
   }
+
   // If embed_mode, then pass on the serialized blob
   // If not embed_mode, dump the blob here and only pass on the path to the blob
   if (global_context_.ep_context_embed_mode) {
     std::ostringstream model_blob_stream;
     compiled_model.export_model(model_blob_stream);
-    model_blob_str = model_blob_stream.str();
-    ORT_ENFORCE(model_blob_str.size() != 0);
+    model_blob_str = std::move(model_blob_stream).str();
+    if (model_blob_str.empty()) {
+      ORT_THROW("Model blob stream is empty after exporting the compiled model.");
+    }
   } else {
     // Remove extension so we can append suffix to form the complete name of output graph
-    auto blob_name = [&]() {
-      size_t dot = graph_name.find_last_of(".");
-      if (dot == std::string::npos) return graph_name;
-      return graph_name.substr(0, dot);
-    }();
+    auto blob_name = graph_name.substr(0, graph_name.find_last_of("."));
     std::ofstream blob_file(blob_name + ".blob",
                             std::ios::out | std::ios::trunc | std::ios::binary);
     if (!blob_file) {
@@ -194,7 +190,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
                                                       graph_name,
                                                       logger,
                                                       global_context_.ep_context_embed_mode,
-                                                      model_blob_str,
+                                                      std::move(model_blob_str),
                                                       openvino_sdk_version_));
 
   return Status::OK();

onnxruntime/core/providers/openvino/backends/basic_backend.cc

@@ -66,7 +66,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
         exe_network_ = global_context_.ie_core.ImportModel(model_stream,
                                                            remote_context_,
                                                            subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else if ((global_context.device_type.find("GPU") != std::string::npos) &&
                  (global_context_.context != nullptr)) {
         LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled";
@@ -75,7 +74,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
             ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else {
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
@@ -91,7 +89,15 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
                                                            device_config,
                                                            global_context_.ep_context_embed_mode,
                                                            subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
+        // ie_cnn_network_ = exe_network_.Get().get_runtime_model();
+      } else if (global_context_.export_ep_ctx_blob &&
+                 hw_target.find("NPU") != std::string::npos) {
+        std::shared_ptr<ov::Model> ov_model;
+        {
+          const std::string model = model_proto.SerializeAsString();
+          ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor());
+        }
+        exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config));
       } else if ((!subgraph_context_.has_dynamic_input_shape) &&
                  ((hw_target.find("AUTO") == std::string::npos) ||
                   (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) {
@@ -102,7 +108,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
                                                             hw_target,
                                                             device_config,
                                                             subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else {  // For all other types use ov::Model Type
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
@@ -270,14 +275,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque
           input_tensor_shape[tensor_iter] = *i;
           tensor_iter += 1;
         }
-        auto input = ie_cnn_network_->get_parameters().at(input_idx);
+        auto input = graph_input_info.at(input_idx);
         OVTensorPtr tensor_ptr;
         // avoid input copies on the CPU device
         if (global_context_.device_type.find("CPU") != std::string::npos) {
-          tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape,
+          tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape,
                                                     (void*)tensor_data);
         } else {
-          tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape);
+          tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape);
           FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_);
         }
 
@@ -341,9 +346,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
         const void* tensor_data = tensor.GetTensorRawData();
         const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
         // Create an Input Remote Blob
-        auto input = ie_cnn_network_->get_parameters().at(0);
+        auto input = graph_input_info.at(0);
         auto remote_blob = remote_context_->create_tensor(
-            input->get_element_type(), input->get_shape(), *shared_buffer_const);
+            input.get_element_type(), input.get_shape(), *shared_buffer_const);
         ov::Tensor tensor_remote = static_cast<ov::Tensor>(remote_blob);
         OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_remote);
         infer_request->SetTensor(input_name, tensor_ptr);
@@ -392,9 +397,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
         const void* tensor_data = tensor.GetTensorRawData();
         const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
         // Create a shared Blob, set the Infer Request Output Blob
-        auto output = ie_cnn_network_->get_results().at(0);
+        auto output = graph_output_info.at(0);
         auto remote_tensor =
-            remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const);
+            remote_context_->create_tensor(output.get_element_type(), output.get_shape(), *shared_buffer_const);
         ov::Tensor tensor_t = static_cast<ov::Tensor>(remote_tensor);
         OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_t);
         try {

onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc

@@ -4,6 +4,7 @@
 #include <string>
 #include <fstream>
 #include <vector>
+#include <algorithm>
 
 #include "core/providers/openvino/onnx_ctx_model_helper.h"
 
@@ -18,71 +19,77 @@ Status EPCtxHandler::ExportEPCtxModel(const GraphViewer& graph_viewer,
                                       const std::string& graph_name,
                                       const logging::Logger& logger,
                                       const bool& ep_context_embed_mode,
-                                      const std::string& model_blob_str,
+                                      std::string&& model_blob_str,
                                       const std::string& openvino_sdk_version) const {
   auto model_build = graph_viewer.CreateModel(logger);
   auto& graph_build = model_build->MainGraph();
 
   // Get graph inputs and outputs
-  std::vector<onnxruntime::NodeArg*> inputs, outputs;
-  for (auto input : graph_viewer.GetInputs()) {
-    auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
-    inputs.push_back(&n_input);
-  }
-  for (auto output : graph_viewer.GetOutputs()) {
-    auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
-    outputs.push_back(&n_output);
-  }
+  const auto& viewer_inputs = graph_viewer.GetInputs();
+  const auto& viewer_outputs = graph_viewer.GetOutputs();
+  std::vector<onnxruntime::NodeArg*> inputs(viewer_inputs.size()), outputs(viewer_outputs.size());
+  auto transform_f = [&](const onnxruntime::NodeArg* iter) { return &graph_build.GetOrCreateNodeArg(iter->Name(), iter->TypeAsProto()); };
+  auto fill_vectors = [transform_f](auto& src, auto& dst) {
+    std::transform(src.begin(), src.end(), dst.begin(), transform_f);
+  };
+  fill_vectors(viewer_inputs, inputs);
+  fill_vectors(viewer_outputs, outputs);
 
   // Create EP context node attributes
-  auto attr_0 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_1 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_2 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_3 = ONNX_NAMESPACE::AttributeProto::Create();
-
-  // embed mode
-  attr_0->set_name(EMBED_MODE);
-  attr_0->set_type(onnx::AttributeProto_AttributeType_INT);
-  attr_0->set_i(ep_context_embed_mode);
-  // ep context
-  attr_1->set_name(EP_CACHE_CONTEXT);
-  attr_1->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_1->set_s(model_blob_str);
-  // sdk version
-  attr_2->set_name(EP_SDK_VER);
-  attr_2->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_2->set_s(openvino_sdk_version);
-  // source
-  attr_3->set_name(SOURCE);
-  attr_3->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_3->set_s(kOpenVINOExecutionProvider);
-
   auto node_attributes = ONNX_NAMESPACE::NodeAttributes::Create();
   node_attributes->reserve(4);
-  node_attributes->emplace(EMBED_MODE, *attr_0);
-  node_attributes->emplace(EP_CACHE_CONTEXT, *attr_1);
-  node_attributes->emplace(EP_SDK_VER, *attr_2);
-  node_attributes->emplace(SOURCE, *attr_3);
-
+  {
+    // Create EP context node attributes
+
+    // embed mode
+    auto embed_mode_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    embed_mode_attr->set_name(EMBED_MODE);
+    embed_mode_attr->set_type(onnx::AttributeProto_AttributeType_INT);
+    embed_mode_attr->set_i(ep_context_embed_mode);
+    node_attributes->emplace(EMBED_MODE, std::move(*embed_mode_attr));
+
+    // ep context
+    auto ep_cache_context_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    ep_cache_context_attr->set_name(EP_CACHE_CONTEXT);
+    ep_cache_context_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    ep_cache_context_attr->set_s(std::move(model_blob_str));
+    node_attributes->emplace(EP_CACHE_CONTEXT, std::move(*ep_cache_context_attr));
+
+    // sdk version
+    auto sdk_version_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    sdk_version_attr->set_name(EP_SDK_VER);
+    sdk_version_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    sdk_version_attr->set_s(openvino_sdk_version);
+    node_attributes->emplace(EP_SDK_VER, std::move(*sdk_version_attr));
+
+    // source
+    auto source_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    source_attr->set_name(SOURCE);
+    source_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    source_attr->set_s(kOpenVINOExecutionProvider);
+    node_attributes->emplace(SOURCE, std::move(*source_attr));
+  }
   // Create EP context node
-  graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, node_attributes.get(), kMSDomain);
+  graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, std::move(*node_attributes), kMSDomain);
   ORT_ENFORCE(graph_build.Resolve().IsOK());
 
-  // Serialize modelproto to string
-  auto new_graph_viewer = graph_build.CreateGraphViewer();
-  auto model = new_graph_viewer->CreateModel(logger);
-  auto model_proto = model->ToProto();
-  new_graph_viewer->ToProto(*model_proto->mutable_graph(), true, true);
-  model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
-
-  // Finally, dump the model
-  std::ofstream epctx_onnx_model(graph_name,
-                                 std::ios::out | std::ios::trunc | std::ios::binary);
-  if (!epctx_onnx_model) {
-    ORT_THROW("Unable to create epctx onnx model file ");
-  }
-  model_proto->SerializeToOstream(epctx_onnx_model);
+  {
+    // Serialize modelproto to string
+    auto model_proto = model_build->ToProto();
+    model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+    // Finally, dump the model
+    std::ofstream epctx_onnx_model(graph_name,
+                                   std::ios::out | std::ios::trunc | std::ios::binary);
+    if (!epctx_onnx_model) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unable to create epctx onnx model file");
+    }
 
+    if (!model_proto->SerializeToOstream(epctx_onnx_model)) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to serialize model to file");
+    }
+    model_proto->SerializeToOstream(epctx_onnx_model);
+  }
   LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Export blob as EPContext Node";
 
   return Status::OK();

onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h

@@ -28,7 +28,7 @@ class EPCtxHandler {
                           const std::string& graph_name,
                           const logging::Logger& logger,
                           const bool& ep_context_embed_mode,
-                          const std::string& model_blob_str,
+                          std::string&& model_blob_str,
                           const std::string& openvino_sdk_version) const;
   Status ImportBlobFromEPCtxModel(const GraphViewer& graph_viewer);
   bool CheckForOVEPCtxNode(const GraphViewer& graph_viewer, std::string openvino_sdk_version) const;