From ecb4626fad8efb1b34891e2e40cc5094b001a9cc Mon Sep 17 00:00:00 2001
From: saurabhkale17 <saurabh1.kale@intel.com>
Date: Sat, 3 Aug 2024 23:24:26 -0700
Subject: [PATCH 1/9] fix handling of multiple QuantizeLinear nodes

---
 .../qdq_transformations/qdq_stripping.cc      | 33 ++++++++++---------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
index a2b3ed068235b..fca681082ef0b 100644
--- a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
+++ b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
@@ -583,22 +583,23 @@ static void AddQDQNodeUnit(onnxruntime::Graph& dst_graph,
 
   // Handle Qs in the NodeUnit
   if (!node_unit.GetQNodes().empty()) {
-    ORT_ENFORCE(node_unit.GetQNodes().size() == 1);
-    const auto& q_node = node_unit.GetQNodes().at(0);
-
-    SkipReason reason;
-
-    bool keep_q = CheckQRuleSet(node_unit, q_node, src_graph, reason);
-
-    if (keep_q) {
-      AddNode(initializers_to_keep, src_graph, dst_graph, *q_node);
-      // if keep_q, then output defs of the target node doesn't change
-      output_args.push_back(&dst_graph.GetOrCreateNodeArg(target_node.OutputDefs().at(0)->Name(),
-                                                          target_node.OutputDefs().at(0)->TypeAsProto()));
-    } else {
-      // convert this Q to float
-      output_args.push_back(&ProcessNodeUnitIO(dst_graph, src_graph, initializers_to_keep,
-                                               node_unit_outputs.at(0)));
+    for (auto i = 0 ; i < node_unit.GetQNodes().size() ; i++) {
+      const auto& q_node = node_unit.GetQNodes().at(0);
+
+      SkipReason reason;
+
+      bool keep_q = CheckQRuleSet(node_unit, q_node, src_graph, reason);
+
+      if (keep_q) {
+        AddNode(initializers_to_keep, src_graph, dst_graph, *q_node);
+        // if keep_q, then output defs of the target node doesn't change
+        output_args.push_back(&dst_graph.GetOrCreateNodeArg(target_node.OutputDefs().at(i)->Name(),
+                                                            target_node.OutputDefs().at(i)->TypeAsProto()));
+      } else {
+        // convert this Q to float
+        output_args.push_back(&ProcessNodeUnitIO(dst_graph, src_graph, initializers_to_keep,
+                                                node_unit_outputs.at(i)));
+      }
     }
   } else {
     for (const auto& node_unit_output : node_unit_outputs) {

From 4d82fa7a15bb0b114d9af8d9d8f5a8a9f73db770 Mon Sep 17 00:00:00 2001
From: saurabhkale17 <saurabh1.kale@intel.com>
Date: Mon, 5 Aug 2024 11:28:54 -0700
Subject: [PATCH 2/9] Fix mutiple Qnodes

---
 .../providers/openvino/qdq_transformations/qdq_stripping.cc     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
index fca681082ef0b..ff88ef9b6f374 100644
--- a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
+++ b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
@@ -584,7 +584,7 @@ static void AddQDQNodeUnit(onnxruntime::Graph& dst_graph,
   // Handle Qs in the NodeUnit
   if (!node_unit.GetQNodes().empty()) {
     for (auto i = 0 ; i < node_unit.GetQNodes().size() ; i++) {
-      const auto& q_node = node_unit.GetQNodes().at(0);
+      const auto& q_node = node_unit.GetQNodes().at(i);
 
       SkipReason reason;
 

From 0e5aa49ce4963ebbd87c96596d23a20dfe626bfe Mon Sep 17 00:00:00 2001
From: saurabhkale17 <saurabh1.kale@intel.com>
Date: Mon, 5 Aug 2024 19:06:35 -0700
Subject: [PATCH 3/9] fix debug build

---
 .../providers/openvino/qdq_transformations/qdq_stripping.cc     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
index ff88ef9b6f374..b1e4c395b9daa 100644
--- a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
+++ b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
@@ -583,7 +583,7 @@ static void AddQDQNodeUnit(onnxruntime::Graph& dst_graph,
 
   // Handle Qs in the NodeUnit
   if (!node_unit.GetQNodes().empty()) {
-    for (auto i = 0 ; i < node_unit.GetQNodes().size() ; i++) {
+    for (size_t i = 0 ; i < node_unit.GetQNodes().size() ; i++) {
       const auto& q_node = node_unit.GetQNodes().at(i);
 
       SkipReason reason;

From 02bdd03e0abe43a3e8c9ca66db0a5b8d47730fe9 Mon Sep 17 00:00:00 2001
From: Preetha Veeramalai <preetha.veeramalai@intel.com>
Date: Fri, 23 Aug 2024 02:24:56 -0700
Subject: [PATCH 4/9] Ovep mem optimization (#422)

* Update onnx_ctx_model_helper.cc

Keep "attr_0" "attr_1" etc inside a scope so that these unique pointers get deleted after their use is over
Also declare "node_attributes" global to the new scope (within the function)

* Optimizations to reduce memory used during compilation

* Fix for reusing the serialized compiled blob to proto model

* EpCtx export code refactoring

* Fix from Ankit to move memory during ONNX Graph creation

* Localise the OV IR model in memory

* Apply lint changes

---------

Co-authored-by: Vishnudas Thaniel S <vishnudas.thaniel.s@intel.com>
Co-authored-by: Javier E. Martinez <javier.e.martinez@intel.com>
---
 include/onnxruntime/core/graph/graph.h        |  14 +++
 onnxruntime/core/graph/graph.cc               |  65 ++++++++++
 .../providers/openvino/backend_manager.cc     |  26 ++--
 .../openvino/backends/basic_backend.cc        |  27 +++--
 .../openvino/onnx_ctx_model_helper.cc         | 111 ++++++++++--------
 .../openvino/onnx_ctx_model_helper.h          |   2 +-
 .../qdq_transformations/qdq_stripping.cc      |   4 +-
 .../shared_library/provider_interfaces.h      |   3 +
 .../shared_library/provider_wrappedtypes.h    |   3 +
 .../core/session/provider_bridge_ort.cc       |   5 +
 10 files changed, 179 insertions(+), 81 deletions(-)

diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h
index c51f38553c3b4..e3298d6c886ad 100644
--- a/include/onnxruntime/core/graph/graph.h
+++ b/include/onnxruntime/core/graph/graph.h
@@ -571,6 +571,13 @@ class Node {
             gsl::span<NodeArg* const> output_args,
             const NodeAttributes* attributes,
             std::string_view domain);
+  void Init(std::string_view name,
+            std::string_view op_type,
+            std::string_view description,
+            gsl::span<NodeArg* const> input_args,
+            gsl::span<NodeArg* const> output_args,
+            NodeAttributes&& attributes,
+            std::string_view domain);
 #endif
 
 #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@@ -952,6 +959,13 @@ class Graph {  // NOLINT(clang-analyzer-optin.performance.Padding): preserve exi
                 const NodeAttributes* attributes = nullptr,
                 const std::string& domain = kOnnxDomain);
 
+  Node& AddNode(const std::string& name,
+                const std::string& op_type,
+                const std::string& description,
+                gsl::span<NodeArg* const> input_args,
+                gsl::span<NodeArg* const> output_args,
+                NodeAttributes&& attributes,
+                const std::string& domain = kOnnxDomain);
   Node& AddNode(const std::string& name,
                 const std::string& op_type,
                 const std::string& description,
diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc
index e950d68947b91..54743d82a8ee2 100644
--- a/onnxruntime/core/graph/graph.cc
+++ b/onnxruntime/core/graph/graph.cc
@@ -915,6 +915,42 @@ void Node::Init(std::string_view name,
     }
   }
 }
+void Node::Init(std::string_view name,
+                std::string_view op_type,
+                std::string_view description,
+                gsl::span<NodeArg* const> input_args,
+                gsl::span<NodeArg* const> output_args,
+                NodeAttributes&& attributes,
+                std::string_view domain) {
+  name_ = name;
+  op_type_ = op_type;
+  description_ = description;
+  definitions_.input_defs.assign(input_args.begin(), input_args.end());
+  definitions_.output_defs.assign(output_args.begin(), output_args.end());
+  domain_ = domain;
+  can_be_saved_ = true;
+  priority_ = 0;
+  if (kOnnxDomainAlias == domain_) {
+    domain_ = kOnnxDomain;
+  }
+
+  // Set each arg count as 1 by default.
+  // It could be adjusted when resolving the node with its operator
+  // information.
+  definitions_.input_arg_count.assign(input_args.size(), 1);
+
+  attributes_ = std::move(attributes);
+
+  for (auto& name_to_attr : attributes_) {
+    if (utils::HasGraph(name_to_attr.second)) {
+#if !defined(ORT_MINIMAL_BUILD)
+      CreateSubgraph(name_to_attr.first);
+#else
+      ORT_THROW("Creating node with a subgraph via AddNode is not supported in this build.");
+#endif
+    }
+  }
+}
 #endif  // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
 
 #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
@@ -3923,6 +3959,35 @@ Node& Graph::AddNode(const std::string& name,
   return *node;
 }
 
+Node& Graph::AddNode(const std::string& name,
+                     const std::string& op_type,
+                     const std::string& description,
+                     gsl::span<NodeArg* const> input_args,
+                     gsl::span<NodeArg* const> output_args,
+                     NodeAttributes&& attributes,
+                     const std::string& domain) {
+  InlinedVector<NodeArg*> inputs;
+  InlinedVector<NodeArg*> outputs;
+  inputs.resize(input_args.size());
+  outputs.resize(output_args.size());
+  int i = 0;
+  for (auto input_arg : input_args) {
+    inputs[i++] = &GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto());
+  }
+  i = 0;
+  for (auto output_arg : output_args) {
+    outputs[i++] = &GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto());
+  }
+
+  const gsl::not_null<Node*> node = AllocateNode();
+  node->Init(name, op_type, description, inputs, outputs, std::move(attributes), domain);
+  if (0 != op_type.compare(kNoOp)) {
+    GraphProtoSyncNeeded(true);
+  }
+
+  return *node;
+}
+
 bool Graph::RemoveNode(NodeIndex p_index) {
   auto node = GetNode(p_index);
   if (nullptr == node) {
diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc
index 18a6257910a56..4d2e38022b66f 100644
--- a/onnxruntime/core/providers/openvino/backend_manager.cc
+++ b/onnxruntime/core/providers/openvino/backend_manager.cc
@@ -155,32 +155,28 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
   auto compiled_model = concrete_backend_->GetOVCompiledModel();
   std::string graph_name = "";
   // Epctx file path from SO is mapped to cache_dir variable for OVEP for readability
-  if (global_context_.cache_dir != "") {
+  if (!global_context_.cache_dir.empty()) {
     graph_name = global_context_.cache_dir;
   } else {
     graph_name = global_context_.onnx_model_path_name;
     // Remove extension so we can append suffix to form the complete name of output graph
-    graph_name = [&]() {
-      size_t dot = graph_name.find_last_of(".");
-      if (dot == std::string::npos) return graph_name;
-      return graph_name.substr(0, dot);
-    }();
-    graph_name = graph_name + "_ctx.onnx";
+    size_t dot = global_context_.onnx_model_path_name.find_last_of(".");
+    graph_name = graph_name.substr(0, dot);
+    if (dot != std::string::npos) graph_name += "_ctx.onnx";
   }
+
   // If embed_mode, then pass on the serialized blob
   // If not embed_mode, dump the blob here and only pass on the path to the blob
   if (global_context_.ep_context_embed_mode) {
     std::ostringstream model_blob_stream;
     compiled_model.export_model(model_blob_stream);
-    model_blob_str = model_blob_stream.str();
-    ORT_ENFORCE(model_blob_str.size() != 0);
+    model_blob_str = std::move(model_blob_stream).str();
+    if (model_blob_str.empty()) {
+      ORT_THROW("Model blob stream is empty after exporting the compiled model.");
+    }
   } else {
     // Remove extension so we can append suffix to form the complete name of output graph
-    auto blob_name = [&]() {
-      size_t dot = graph_name.find_last_of(".");
-      if (dot == std::string::npos) return graph_name;
-      return graph_name.substr(0, dot);
-    }();
+    auto blob_name = graph_name.substr(0, graph_name.find_last_of("."));
     std::ofstream blob_file(blob_name + ".blob",
                             std::ios::out | std::ios::trunc | std::ios::binary);
     if (!blob_file) {
@@ -194,7 +190,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
                                                       graph_name,
                                                       logger,
                                                       global_context_.ep_context_embed_mode,
-                                                      model_blob_str,
+                                                      std::move(model_blob_str),
                                                       openvino_sdk_version_));
 
   return Status::OK();
diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc
index d79aa35be6418..0ee2926ce1fcc 100644
--- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc
+++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc
@@ -66,7 +66,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
         exe_network_ = global_context_.ie_core.ImportModel(model_stream,
                                                            remote_context_,
                                                            subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else if ((global_context.device_type.find("GPU") != std::string::npos) &&
                  (global_context_.context != nullptr)) {
         LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled";
@@ -75,7 +74,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
             ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else {
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
@@ -91,7 +89,15 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
                                                            device_config,
                                                            global_context_.ep_context_embed_mode,
                                                            subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
+        // ie_cnn_network_ = exe_network_.Get().get_runtime_model();
+      } else if (global_context_.export_ep_ctx_blob &&
+                 hw_target.find("NPU") != std::string::npos) {
+        std::shared_ptr<ov::Model> ov_model;
+        {
+          const std::string model = model_proto.SerializeAsString();
+          ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor());
+        }
+        exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config));
       } else if ((!subgraph_context_.has_dynamic_input_shape) &&
                  ((hw_target.find("AUTO") == std::string::npos) ||
                   (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) {
@@ -102,7 +108,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto,
                                                             hw_target,
                                                             device_config,
                                                             subgraph_context_.subgraph_name);
-        ie_cnn_network_ = exe_network_.Get().get_runtime_model();
       } else {  // For all other types use ov::Model Type
         ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_);
         exe_network_ = global_context_.ie_core.CompileModel(
@@ -270,14 +275,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque
           input_tensor_shape[tensor_iter] = *i;
           tensor_iter += 1;
         }
-        auto input = ie_cnn_network_->get_parameters().at(input_idx);
+        auto input = graph_input_info.at(input_idx);
         OVTensorPtr tensor_ptr;
         // avoid input copies on the CPU device
         if (global_context_.device_type.find("CPU") != std::string::npos) {
-          tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape,
+          tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape,
                                                     (void*)tensor_data);
         } else {
-          tensor_ptr = std::make_shared<ov::Tensor>(input->get_element_type(), input_tensor_shape);
+          tensor_ptr = std::make_shared<ov::Tensor>(input.get_element_type(), input_tensor_shape);
           FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_);
         }
 
@@ -341,9 +346,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
         const void* tensor_data = tensor.GetTensorRawData();
         const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
         // Create an Input Remote Blob
-        auto input = ie_cnn_network_->get_parameters().at(0);
+        auto input = graph_input_info.at(0);
         auto remote_blob = remote_context_->create_tensor(
-            input->get_element_type(), input->get_shape(), *shared_buffer_const);
+            input.get_element_type(), input.get_shape(), *shared_buffer_const);
         ov::Tensor tensor_remote = static_cast<ov::Tensor>(remote_blob);
         OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_remote);
         infer_request->SetTensor(input_name, tensor_ptr);
@@ -392,9 +397,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe
         const void* tensor_data = tensor.GetTensorRawData();
         const cl::Buffer* shared_buffer_const = static_cast<const cl::Buffer*>(tensor_data);
         // Create a shared Blob, set the Infer Request Output Blob
-        auto output = ie_cnn_network_->get_results().at(0);
+        auto output = graph_output_info.at(0);
         auto remote_tensor =
-            remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const);
+            remote_context_->create_tensor(output.get_element_type(), output.get_shape(), *shared_buffer_const);
         ov::Tensor tensor_t = static_cast<ov::Tensor>(remote_tensor);
         OVTensorPtr tensor_ptr = std::make_shared<ov::Tensor>(tensor_t);
         try {
diff --git a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
index e2df9c83f15ae..49c009ac60178 100644
--- a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
+++ b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
@@ -4,6 +4,7 @@
 #include <string>
 #include <fstream>
 #include <vector>
+#include <algorithm>
 
 #include "core/providers/openvino/onnx_ctx_model_helper.h"
 
@@ -18,71 +19,77 @@ Status EPCtxHandler::ExportEPCtxModel(const GraphViewer& graph_viewer,
                                       const std::string& graph_name,
                                       const logging::Logger& logger,
                                       const bool& ep_context_embed_mode,
-                                      const std::string& model_blob_str,
+                                      std::string&& model_blob_str,
                                       const std::string& openvino_sdk_version) const {
   auto model_build = graph_viewer.CreateModel(logger);
   auto& graph_build = model_build->MainGraph();
 
   // Get graph inputs and outputs
-  std::vector<onnxruntime::NodeArg*> inputs, outputs;
-  for (auto input : graph_viewer.GetInputs()) {
-    auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto());
-    inputs.push_back(&n_input);
-  }
-  for (auto output : graph_viewer.GetOutputs()) {
-    auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto());
-    outputs.push_back(&n_output);
-  }
+  const auto& viewer_inputs = graph_viewer.GetInputs();
+  const auto& viewer_outputs = graph_viewer.GetOutputs();
+  std::vector<onnxruntime::NodeArg*> inputs(viewer_inputs.size()), outputs(viewer_outputs.size());
+  auto transform_f = [&](const onnxruntime::NodeArg* iter) { return &graph_build.GetOrCreateNodeArg(iter->Name(), iter->TypeAsProto()); };
+  auto fill_vectors = [transform_f](auto& src, auto& dst) {
+    std::transform(src.begin(), src.end(), dst.begin(), transform_f);
+  };
+  fill_vectors(viewer_inputs, inputs);
+  fill_vectors(viewer_outputs, outputs);
 
   // Create EP context node attributes
-  auto attr_0 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_1 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_2 = ONNX_NAMESPACE::AttributeProto::Create();
-  auto attr_3 = ONNX_NAMESPACE::AttributeProto::Create();
-
-  // embed mode
-  attr_0->set_name(EMBED_MODE);
-  attr_0->set_type(onnx::AttributeProto_AttributeType_INT);
-  attr_0->set_i(ep_context_embed_mode);
-  // ep context
-  attr_1->set_name(EP_CACHE_CONTEXT);
-  attr_1->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_1->set_s(model_blob_str);
-  // sdk version
-  attr_2->set_name(EP_SDK_VER);
-  attr_2->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_2->set_s(openvino_sdk_version);
-  // source
-  attr_3->set_name(SOURCE);
-  attr_3->set_type(onnx::AttributeProto_AttributeType_STRING);
-  attr_3->set_s(kOpenVINOExecutionProvider);
-
   auto node_attributes = ONNX_NAMESPACE::NodeAttributes::Create();
   node_attributes->reserve(4);
-  node_attributes->emplace(EMBED_MODE, *attr_0);
-  node_attributes->emplace(EP_CACHE_CONTEXT, *attr_1);
-  node_attributes->emplace(EP_SDK_VER, *attr_2);
-  node_attributes->emplace(SOURCE, *attr_3);
-
+  {
+    // Create EP context node attributes
+
+    // embed mode
+    auto embed_mode_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    embed_mode_attr->set_name(EMBED_MODE);
+    embed_mode_attr->set_type(onnx::AttributeProto_AttributeType_INT);
+    embed_mode_attr->set_i(ep_context_embed_mode);
+    node_attributes->emplace(EMBED_MODE, std::move(*embed_mode_attr));
+
+    // ep context
+    auto ep_cache_context_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    ep_cache_context_attr->set_name(EP_CACHE_CONTEXT);
+    ep_cache_context_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    ep_cache_context_attr->set_s(std::move(model_blob_str));
+    node_attributes->emplace(EP_CACHE_CONTEXT, std::move(*ep_cache_context_attr));
+
+    // sdk version
+    auto sdk_version_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    sdk_version_attr->set_name(EP_SDK_VER);
+    sdk_version_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    sdk_version_attr->set_s(openvino_sdk_version);
+    node_attributes->emplace(EP_SDK_VER, std::move(*sdk_version_attr));
+
+    // source
+    auto source_attr = ONNX_NAMESPACE::AttributeProto::Create();
+    source_attr->set_name(SOURCE);
+    source_attr->set_type(onnx::AttributeProto_AttributeType_STRING);
+    source_attr->set_s(kOpenVINOExecutionProvider);
+    node_attributes->emplace(SOURCE, std::move(*source_attr));
+  }
   // Create EP context node
-  graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, node_attributes.get(), kMSDomain);
+  graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, std::move(*node_attributes), kMSDomain);
   ORT_ENFORCE(graph_build.Resolve().IsOK());
 
-  // Serialize modelproto to string
-  auto new_graph_viewer = graph_build.CreateGraphViewer();
-  auto model = new_graph_viewer->CreateModel(logger);
-  auto model_proto = model->ToProto();
-  new_graph_viewer->ToProto(*model_proto->mutable_graph(), true, true);
-  model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
-
-  // Finally, dump the model
-  std::ofstream epctx_onnx_model(graph_name,
-                                 std::ios::out | std::ios::trunc | std::ios::binary);
-  if (!epctx_onnx_model) {
-    ORT_THROW("Unable to create epctx onnx model file ");
-  }
-  model_proto->SerializeToOstream(epctx_onnx_model);
+  {
+    // Serialize modelproto to string
+    auto model_proto = model_build->ToProto();
+    model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION);
+
+    // Finally, dump the model
+    std::ofstream epctx_onnx_model(graph_name,
+                                   std::ios::out | std::ios::trunc | std::ios::binary);
+    if (!epctx_onnx_model) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unable to create epctx onnx model file");
+    }
 
+    if (!model_proto->SerializeToOstream(epctx_onnx_model)) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to serialize model to file");
+    }
+    model_proto->SerializeToOstream(epctx_onnx_model);
+  }
   LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Export blob as EPContext Node";
 
   return Status::OK();
diff --git a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h
index 610e9fd49c901..c631d011d02b1 100644
--- a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h
+++ b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h
@@ -28,7 +28,7 @@ class EPCtxHandler {
                           const std::string& graph_name,
                           const logging::Logger& logger,
                           const bool& ep_context_embed_mode,
-                          const std::string& model_blob_str,
+                          std::string&& model_blob_str,
                           const std::string& openvino_sdk_version) const;
   Status ImportBlobFromEPCtxModel(const GraphViewer& graph_viewer);
   bool CheckForOVEPCtxNode(const GraphViewer& graph_viewer, std::string openvino_sdk_version) const;
diff --git a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
index b1e4c395b9daa..f1df1abf4c49a 100644
--- a/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
+++ b/onnxruntime/core/providers/openvino/qdq_transformations/qdq_stripping.cc
@@ -583,7 +583,7 @@ static void AddQDQNodeUnit(onnxruntime::Graph& dst_graph,
 
   // Handle Qs in the NodeUnit
   if (!node_unit.GetQNodes().empty()) {
-    for (size_t i = 0 ; i < node_unit.GetQNodes().size() ; i++) {
+    for (size_t i = 0; i < node_unit.GetQNodes().size(); i++) {
       const auto& q_node = node_unit.GetQNodes().at(i);
 
       SkipReason reason;
@@ -598,7 +598,7 @@ static void AddQDQNodeUnit(onnxruntime::Graph& dst_graph,
       } else {
         // convert this Q to float
         output_args.push_back(&ProcessNodeUnitIO(dst_graph, src_graph, initializers_to_keep,
-                                                node_unit_outputs.at(i)));
+                                                 node_unit_outputs.at(i)));
       }
     }
   } else {
diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h
index a9394838aa784..3e6a8b87ca646 100644
--- a/onnxruntime/core/providers/shared_library/provider_interfaces.h
+++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h
@@ -379,6 +379,7 @@ struct ProviderHost {
   virtual float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) = 0;
   virtual const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) = 0;
   virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) = 0;
+  virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) = 0;
   virtual void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) = 0;
   virtual void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) = 0;
   virtual void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0;
@@ -849,6 +850,7 @@ struct ProviderHost {
   virtual std::unique_ptr<NodeAttributes_Iterator> NodeAttributes__find(const NodeAttributes* p, const std::string& key) = 0;
   virtual void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) = 0;
   virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0;
+  virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) = 0;
   virtual void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0;
   virtual void NodeAttributes__reserve(NodeAttributes* p, size_t size) = 0;
 
@@ -900,6 +902,7 @@ struct ProviderHost {
   virtual Status Graph__Resolve(Graph* p) = 0;
   virtual void Graph__AddInitializedTensor(Graph* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0;
   virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, const NodeAttributes* attributes, const std::string& domain) = 0;
+  virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, NodeAttributes&& attributes, const std::string& domain) = 0;
   virtual Node& Graph__AddNode(Graph* p, const Node& other) = 0;
 
   virtual const std::vector<const NodeArg*>& Graph__GetOutputs(const Graph* p) noexcept = 0;
diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
index 242c7126f3274..cdf8122e23451 100644
--- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
+++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
@@ -114,6 +114,7 @@ struct AttributeProto final {
   float f() const { return g_host->AttributeProto__f(this); }
   const ONNX_NAMESPACE::TensorProto& t() const { return g_host->AttributeProto__t(this); }
   void set_s(const ::std::string& value) { return g_host->AttributeProto__set_s(this, value); }
+  void set_s(::std::string&& value) { return g_host->AttributeProto__set_s(this, ::std::move(value)); }
   void set_f(const float& value) { return g_host->AttributeProto__set_f(this, value); }
   void set_i(int64_t value) { return g_host->AttributeProto__set_i(this, value); }
   void set_t(const TensorProto& value) { return g_host->AttributeProto__set_t(this, value); }
@@ -862,6 +863,7 @@ struct NodeAttributes final {
   IteratorHolder<NodeAttributes_Iterator, std::pair<const std::string, ONNX_NAMESPACE::AttributeProto>> find(const std::string& key) const { return g_host->NodeAttributes__find(this, key); }
   void insert(const NodeAttributes& v) { return g_host->NodeAttributes__insert(this, v); }
   void emplace(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__emplace(this, k, v); }
+  void emplace(const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) { g_host->NodeAttributes__emplace(this, k, std::move(v)); }
   void insert_or_assign(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__insert_or_assign(this, k, v); }
 
   void reserve(size_t size) { g_host->NodeAttributes__reserve(this, size); }
@@ -947,6 +949,7 @@ struct Graph final {
   Status Resolve() { return g_host->Graph__Resolve(this); }
   void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor) { return g_host->Graph__AddInitializedTensor(this, tensor); }
   Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span<NodeArg* const> input_args, gsl::span<NodeArg* const> output_args, const NodeAttributes* attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, attributes, domain); }
+  Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span<NodeArg* const> input_args, gsl::span<NodeArg* const> output_args, NodeAttributes&& attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, std::move(attributes), domain); }
   Node& AddNode(const Node& other) { return g_host->Graph__AddNode(this, other); }
 
   const std::vector<const NodeArg*>& GetOutputs() const noexcept { return g_host->Graph__GetOutputs(this); }
diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc
index 62e9cfe24c367..562a91a66fa5b 100644
--- a/onnxruntime/core/session/provider_bridge_ort.cc
+++ b/onnxruntime/core/session/provider_bridge_ort.cc
@@ -487,6 +487,7 @@ struct ProviderHostImpl : ProviderHost {
   float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) override { return p->f(); }
   const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) override { return p->t(); }
   void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) override { return p->set_s(value); }
+  void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) override { return p->set_s(::std::move(value)); }
   void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) override { return p->set_f(value); }
   void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) override { return p->set_i(value); }
   void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& value) override { *p->mutable_t() = value; }
@@ -1112,6 +1113,7 @@ struct ProviderHostImpl : ProviderHost {
   }
   void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) override { return p->insert(v.begin(), v.end()); }
   void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->emplace(k, v); }
+  void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) override { p->emplace(k, std::move(v)); }
   void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->insert_or_assign(k, v); }
   void NodeAttributes__reserve(NodeAttributes* p, size_t size) override { p->reserve(size); }
 
@@ -1189,6 +1191,9 @@ struct ProviderHostImpl : ProviderHost {
   Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, const NodeAttributes* attributes, const std::string& domain) override {
     return p->AddNode(name, op_type, description, input_args, output_args, attributes, domain);
   }
+  Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span<NodeArg* const>& input_args, const gsl::span<NodeArg* const>& output_args, NodeAttributes&& attributes, const std::string& domain) override {
+    return p->AddNode(name, op_type, description, input_args, output_args, ::std::move(attributes), domain);
+  }
   Node& Graph__AddNode(Graph* p, const Node& other) override {
     return p->AddNode(other);
   }

From 67184e5b61219a9189947c76c7273de12e2fc4d9 Mon Sep 17 00:00:00 2001
From: jatinwadhwa921 <110383850+jatinwadhwa921@users.noreply.github.com>
Date: Fri, 23 Aug 2024 03:34:40 -0700
Subject: [PATCH 5/9] Fix related to EP context file path (#423)

---
 .../openvino/openvino_provider_factory.cc     | 40 +++++++------------
 1 file changed, 15 insertions(+), 25 deletions(-)

diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
index 4a13071499368..077ecc717502f 100644
--- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
+++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
@@ -30,10 +30,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
         so_epctx_embed_mode_(so_epctx_embed_mode) {
     device_type_ = (device_type == nullptr) ? "" : device_type;
     cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir;
-    if (cache_dir != nullptr) {
-      free(const_cast<void*>(static_cast<const void*>(cache_dir)));
-      cache_dir = nullptr;
-    }
   }
 
   ~OpenVINOProviderFactory() override {
@@ -94,7 +90,7 @@ struct OpenVINO_Provider : Provider {
                                              // speeds up the model's compilation to NPU device specific format.
     int num_of_threads = 0;                  // [num_of_threads]: Overrides the accelerator default value of number of
                                              //  threads with this value at runtime.
-    const char* cache_dir = nullptr;         // [cache_dir]: specify the path to
+    std::string cache_dir = "";              // [cache_dir]: specify the path to
                                              // dump and load the blobs for the model caching/kernel caching (GPU)
                                              // feature. If blob files are already present, it will be directly loaded.
     const char* model_priority = "DEFAULT";  // High-level OpenVINO model priority hint
@@ -186,7 +182,7 @@ struct OpenVINO_Provider : Provider {
     }
 
     if (provider_options_map.find("cache_dir") != provider_options_map.end()) {
-      cache_dir = provider_options_map.at("cache_dir").c_str();
+      cache_dir = provider_options_map.at("cache_dir");
     }
 
     if (provider_options_map.find("context") != provider_options_map.end()) {
@@ -305,26 +301,20 @@ struct OpenVINO_Provider : Provider {
     if (provider_options_map.find("so_epctx_path") != provider_options_map.end()) {
       // The path to dump epctx model is valid only when epctx is enabled.
       // Overrides the cache_dir option to dump model cache files from OV.
-      if (export_ep_ctx_blob) {
-        auto ep_context_file_path_ = provider_options_map.at("so_epctx_path");
-        auto file_path = std::filesystem::path(ep_context_file_path_);
+      if (export_ep_ctx_blob &&
+          !provider_options_map.at("so_epctx_path").empty()) {
+        cache_dir = provider_options_map.at("so_epctx_path");
+        auto file_path = std::filesystem::path(cache_dir);
         // ep_context_file_path_ file extension must be .onnx
-        if (!ep_context_file_path_.empty()) {
-          if (file_path.extension().generic_string() == ".onnx") {
-            // ep_context_file_path_ must be provided as a directory, create it if doesn't exist
-            auto parent_path = file_path.parent_path();
-            if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) &&
-                !std::filesystem::create_directory(parent_path)) {
-              ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n");
-            }
-#ifdef _WIN32
-            cache_dir = _strdup(ep_context_file_path_.c_str());
-#else
-            cache_dir = strdup(ep_context_file_path_.c_str());
-#endif
-          } else {
-            ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + ep_context_file_path_ + " \n");
+        if (file_path.extension().generic_string() == ".onnx") {
+          // ep_context_file_path_ must be provided as a directory, create it if doesn't exist
+          auto parent_path = file_path.parent_path();
+          if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) &&
+              !std::filesystem::create_directory(parent_path)) {
+            ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n");
           }
+        } else {
+          ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir + " \n");
         }
       }
     }
@@ -333,7 +323,7 @@ struct OpenVINO_Provider : Provider {
                                                      const_cast<char*>(precision.c_str()),
                                                      enable_npu_fast_compile,
                                                      num_of_threads,
-                                                     cache_dir,
+                                                     const_cast<char*>(cache_dir.c_str()),
                                                      model_priority,
                                                      num_streams,
                                                      context,

From 48c464b1fa6d356edc383ef587963533cc358117 Mon Sep 17 00:00:00 2001
From: Vishnudas Thaniel S <vishnudas.thaniel.s@intel.com>
Date: Fri, 23 Aug 2024 16:16:39 +0530
Subject: [PATCH 6/9] Update Dockerfile.ubuntu_openvino

Updated the Version of OV to 2024.3
---
 .../github/linux/docker/Dockerfile.ubuntu_openvino        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
index 4382e12a1cd6c..5f525c1310412 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino
@@ -1,7 +1,7 @@
 ARG UBUNTU_VERSION=22.04
 FROM ubuntu:${UBUNTU_VERSION}
 
-ARG OPENVINO_VERSION=2024.0.0
+ARG OPENVINO_VERSION=2024.3.0
 ARG PYTHON_VERSION=3.10
 
 ADD scripts /tmp/scripts
@@ -19,9 +19,9 @@ ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN cd /opt && mkdir -p intel && cd intel && \
-    wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/linux/l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \
-    tar xzf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \
-    mv l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64 openvino_2024.0.0 && \
+    wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \
+    tar xzf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \
+    mv l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64 openvino_2024.3.0 && \
     cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y
 
 WORKDIR /root

From d7296924c6d6eb456ed1aa80e17aced2c504c07a Mon Sep 17 00:00:00 2001
From: ankitm3k <ankit.maheshkar@intel.com>
Date: Fri, 23 Aug 2024 22:03:15 +0530
Subject: [PATCH 7/9] update: Enable C++ 20 Standard for the build

---
 cmake/onnxruntime_providers_openvino.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake
index 5d1a481d40abc..e559583fae8f5 100644
--- a/cmake/onnxruntime_providers_openvino.cmake
+++ b/cmake/onnxruntime_providers_openvino.cmake
@@ -36,6 +36,7 @@
   onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx)
   install(FILES ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino/openvino_provider_factory.h
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/)
+  set_target_properties(onnxruntime_providers_openvino PROPERTIES CXX_STANDARD 20)
   set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX)
   set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime")
   if(NOT MSVC)

From 86eb1f4a922c3a5e73ff3eb85563774024e53c91 Mon Sep 17 00:00:00 2001
From: Preetha Veeramalai <preetha.veeramalai@intel.com>
Date: Mon, 26 Aug 2024 10:53:08 +0530
Subject: [PATCH 8/9] Remove redundant file write in EPCtx model dump

---
 onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
index 49c009ac60178..ee9486a62ea37 100644
--- a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
+++ b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc
@@ -88,7 +88,6 @@ Status EPCtxHandler::ExportEPCtxModel(const GraphViewer& graph_viewer,
     if (!model_proto->SerializeToOstream(epctx_onnx_model)) {
       return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to serialize model to file");
     }
-    model_proto->SerializeToOstream(epctx_onnx_model);
   }
   LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Export blob as EPContext Node";
 

From 6a910cfaa6a929664924a7da21a74a3ea48f72a8 Mon Sep 17 00:00:00 2001
From: Preetha Veeramalai <preetha.veeramalai@intel.com>
Date: Tue, 27 Aug 2024 15:01:01 +0530
Subject: [PATCH 9/9] Remove invalid provider option 'enable_npu_fast_compile'
 in OVEP

---
 .../core/session/onnxruntime_c_api.h          |  2 --
 .../core/providers/openvino/contexts.h        |  1 -
 .../openvino/openvino_execution_provider.cc   |  1 -
 .../openvino/openvino_execution_provider.h    |  4 +--
 .../openvino/openvino_provider_factory.cc     | 34 ++++++-------------
 .../core/session/provider_bridge_ort.cc       |  7 ----
 .../python/onnxruntime_pybind_state.cc        |  9 -----
 .../test/perftest/command_args_parser.cc      |  3 +-
 onnxruntime/test/perftest/ort_test_session.cc | 21 +++---------
 9 files changed, 16 insertions(+), 66 deletions(-)

diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h
index 4674db42fb1c9..5a5b35a4261e4 100644
--- a/include/onnxruntime/core/session/onnxruntime_c_api.h
+++ b/include/onnxruntime/core/session/onnxruntime_c_api.h
@@ -631,7 +631,6 @@ typedef struct OrtMIGraphXProviderOptions {
 typedef struct OrtOpenVINOProviderOptions {
 #ifdef __cplusplus
   OrtOpenVINOProviderOptions() : device_type{},
-                                 enable_npu_fast_compile{},
                                  device_id{},
                                  num_of_threads{},
                                  cache_dir{},
@@ -644,7 +643,6 @@ typedef struct OrtOpenVINOProviderOptions {
    * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16"
    */
   const char* device_type;
-  unsigned char enable_npu_fast_compile;  ///< 0 = disabled, nonzero = enabled
   const char* device_id;
   size_t num_of_threads;  ///< 0 = Use default number of threads
   const char* cache_dir;  // path is set to empty by default
diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h
index 598e985676f8d..91f195ed9144f 100644
--- a/onnxruntime/core/providers/openvino/contexts.h
+++ b/onnxruntime/core/providers/openvino/contexts.h
@@ -15,7 +15,6 @@ namespace openvino_ep {
 struct GlobalContext {
   OVCore ie_core;
   bool is_wholly_supported_graph = false;
-  bool enable_npu_fast_compile = false;
   bool enable_opencl_throttling = false;
   bool disable_dynamic_shapes = false;
   bool ep_context_embed_mode = true;
diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
index 29c45916795d3..9f7775e05ad7d 100644
--- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
+++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc
@@ -22,7 +22,6 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv
   global_context_ = std::make_unique<openvino_ep::GlobalContext>();
   global_context_->device_type = info.device_type_;
   global_context_->precision_str = info.precision_;
-  global_context_->enable_npu_fast_compile = info.enable_npu_fast_compile_;
   global_context_->cache_dir = info.cache_dir_;
   global_context_->model_priority = info.model_priority_;
   global_context_->num_streams = info.num_streams_;
diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h
index 030e5bba71b67..e2f80511dcca1 100644
--- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h
+++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h
@@ -79,7 +79,6 @@ static std::vector<std::string> parseDevices(const std::string& device_string,
 struct OpenVINOExecutionProviderInfo {
   std::string device_type_{""};
   std::string precision_{""};
-  bool enable_npu_fast_compile_{false};
   size_t num_of_threads_{0};
   std::string cache_dir_{""};
   std::string model_priority_{""};
@@ -95,14 +94,13 @@ struct OpenVINOExecutionProviderInfo {
   OpenVINOExecutionProviderInfo() = delete;
 
   explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision,
-                                         bool enable_npu_fast_compile, size_t num_of_threads,
+                                         size_t num_of_threads,
                                          const std::string& cache_dir, const std::string& model_priority,
                                          int num_streams, void* context, bool enable_opencl_throttling,
                                          bool disable_dynamic_shapes, bool export_ep_ctx_blob,
                                          bool enable_qdq_optimizer, bool disable_cpu_fallback,
                                          bool so_epctx_embed_mode)
       : precision_(std::move(precision)),
-        enable_npu_fast_compile_(enable_npu_fast_compile),
         num_of_threads_(num_of_threads),
         cache_dir_(std::move(cache_dir)),
         model_priority_(std::move(model_priority)),
diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
index 077ecc717502f..8e209a5acea4f 100644
--- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
+++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc
@@ -9,7 +9,7 @@
 namespace onnxruntime {
 struct OpenVINOProviderFactory : IExecutionProviderFactory {
   OpenVINOProviderFactory(const char* device_type, const char* precision,
-                          bool enable_npu_fast_compile, size_t num_of_threads,
+                          size_t num_of_threads,
                           const char* cache_dir, const char* model_priority,
                           int num_streams, void* context,
                           bool enable_opencl_throttling, bool disable_dynamic_shapes,
@@ -17,7 +17,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
                           bool disable_cpu_fallback,
                           bool so_epctx_embed_mode)
       : precision_(precision),
-        enable_npu_fast_compile_(enable_npu_fast_compile),
         num_of_threads_(num_of_threads),
         model_priority_(model_priority),
         num_streams_(num_streams),
@@ -40,7 +39,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
  private:
   std::string device_type_;
   std::string precision_;
-  bool enable_npu_fast_compile_;
   size_t num_of_threads_;
   std::string cache_dir_;
   std::string model_priority_;
@@ -55,7 +53,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory {
 };
 
 std::unique_ptr<IExecutionProvider> OpenVINOProviderFactory::CreateProvider() {
-  OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_,
+  OpenVINOExecutionProviderInfo info(device_type_, precision_, num_of_threads_,
                                      cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_,
                                      disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_,
                                      disable_cpu_fallback_,
@@ -86,8 +84,6 @@ struct OpenVINO_Provider : Provider {
                                              // Not setting precision will execute with optimized precision for
                                              // best inference latency. set Precision=ACCURACY for executing models
                                              // with input precision for best accuracy.
-    bool enable_npu_fast_compile = false;    // [enable_npu_fast_compile]: Fast-compile may be optionally enabled to
-                                             // speeds up the model's compilation to NPU device specific format.
     int num_of_threads = 0;                  // [num_of_threads]: Overrides the accelerator default value of number of
                                              //  threads with this value at runtime.
     std::string cache_dir = "";              // [cache_dir]: specify the path to
@@ -102,16 +98,16 @@ struct OpenVINO_Provider : Provider {
                                              // with this value at runtime.
     bool enable_opencl_throttling = false;   // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU
                                              // device (Reduces CPU Utilization when using GPU)
-    bool export_ep_ctx_blob = false;         // Whether to export the pre-compiled blob as an EPContext model.
+    bool export_ep_ctx_blob = false;         // [Internal] Its linked to session_options.ep_context_enable
+                                             // Whether to export the pre-compiled blob as an EPContext model.
+    bool so_epctx_embed_mode = true;         // [Internal] Its linked to session_options.ep_context_filepath
+    void* context = nullptr;                 // Address for IO Buffer
 
-    void* context = nullptr;
+    bool enable_qdq_optimizer = false;  // Enables QDQ pruning for efficient inference latency with NPU
 
-    bool enable_qdq_optimizer = false;
-
-    bool disable_cpu_fallback = false;
-
-    bool so_epctx_embed_mode = true;
+    bool disable_cpu_fallback = false;  // Disable automatic fallback from NPU to CPU during compilation failures
 
+    std::string bool_flag = "";
     if (provider_options_map.find("device_type") != provider_options_map.end()) {
       device_type = provider_options_map.at("device_type").c_str();
 
@@ -224,16 +220,6 @@ struct OpenVINO_Provider : Provider {
                               << "Executing with num_streams=1";
       }
     }
-    std::string bool_flag = "";
-    if (provider_options_map.find("enable_npu_fast_compile") != provider_options_map.end()) {
-      bool_flag = provider_options_map.at("enable_npu_fast_compile");
-      if (bool_flag == "true" || bool_flag == "True")
-        enable_npu_fast_compile = true;
-      else if (bool_flag == "false" || bool_flag == "False")
-        enable_npu_fast_compile = false;
-      bool_flag = "";
-    }
-
     if (provider_options_map.find("enable_opencl_throttling") != provider_options_map.end()) {
       bool_flag = provider_options_map.at("enable_opencl_throttling");
       if (bool_flag == "true" || bool_flag == "True")
@@ -271,6 +257,7 @@ struct OpenVINO_Provider : Provider {
           disable_dynamic_shapes = false;
         }
       }
+      bool_flag = "";
     }
     if (provider_options_map.find("so_export_ep_ctx_blob") != provider_options_map.end()) {
       bool_flag = provider_options_map.at("so_export_ep_ctx_blob");
@@ -321,7 +308,6 @@ struct OpenVINO_Provider : Provider {
 
     return std::make_shared<OpenVINOProviderFactory>(const_cast<char*>(device_type.c_str()),
                                                      const_cast<char*>(precision.c_str()),
-                                                     enable_npu_fast_compile,
                                                      num_of_threads,
                                                      const_cast<char*>(cache_dir.c_str()),
                                                      model_priority,
diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc
index a1da813af2454..5c673e24eeee5 100644
--- a/onnxruntime/core/session/provider_bridge_ort.cc
+++ b/onnxruntime/core/session/provider_bridge_ort.cc
@@ -1893,12 +1893,6 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O
   if (legacy_ov_options->device_type != nullptr)
     ov_options_converted_map["device_type"] = legacy_ov_options->device_type;
 
-  if (legacy_ov_options->enable_npu_fast_compile) {
-    ov_options_converted_map["enable_npu_fast_compile"] = "false";
-  } else {
-    ov_options_converted_map["enable_npu_fast_compile"] = "true";
-  }
-
   if (legacy_ov_options->num_of_threads != '\0')
     ov_options_converted_map["num_of_threads"] = std::to_string(legacy_ov_options->num_of_threads);
 
@@ -1921,7 +1915,6 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O
 
   // Add new provider option below
   ov_options_converted_map["num_streams"] = "1";
-  ov_options_converted_map["export_ep_ctx_blob"] = "false";
   ov_options_converted_map["model_priority"] = "DEFAULT";
   ov_options_converted_map["enable_qdq_optimizer"] = "false";
   return ov_options_converted_map;
diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc
index 47b8d75f22aea..311009157a69c 100644
--- a/onnxruntime/python/onnxruntime_pybind_state.cc
+++ b/onnxruntime/python/onnxruntime_pybind_state.cc
@@ -1045,12 +1045,6 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
         } else if (option.first == "precision") {
           OV_provider_options_map[option.first] = option.second;
           continue;
-        } else if (option.first == "enable_npu_fast_compile") {
-          if (!(option.second == "True" || option.second == "true" ||
-                option.second == "False" || option.second == "false")) {
-            ORT_THROW("Invalid value passed for enable_npu_fast_compile: ", option.second);
-          }
-          OV_provider_options_map[option.first] = option.second;
         } else if (option.first == "enable_opencl_throttling") {
           if (!(option.second == "True" || option.second == "true" ||
                 option.second == "False" || option.second == "false")) {
@@ -1092,9 +1086,6 @@ std::unique_ptr<IExecutionProvider> CreateExecutionProviderInstance(
         } else if (option.first == "context") {
           OV_provider_options_map[option.first] = option.second;
           continue;
-        } else if (option.first == "export_ep_ctx_blob") {
-          OV_provider_options_map[option.first] = option.second;
-          continue;
         } else if (option.first == "enable_qdq_optimizer") {
           OV_provider_options_map[option.first] = option.second;
           continue;
diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc
index 84c3bc16346f3..801767d38323c 100644
--- a/onnxruntime/test/perftest/command_args_parser.cc
+++ b/onnxruntime/test/perftest/command_args_parser.cc
@@ -74,11 +74,10 @@ namespace perftest {
       "\n"
       "\t    [OpenVINO only] [device_type]: Overrides the accelerator hardware type and precision with these values at runtime.\n"
       "\t    [OpenVINO only] [device_id]: Selects a particular hardware device for inference.\n"
-      "\t    [OpenVINO only] [enable_npu_fast_compile]: Optionally enabled to speeds up the model's compilation on NPU device targets.\n"
       "\t    [OpenVINO only] [num_of_threads]: Overrides the accelerator hardware type and precision with these values at runtime.\n"
       "\t    [OpenVINO only] [cache_dir]: Explicitly specify the path to dump and load the blobs(Model caching) or cl_cache (Kernel Caching) files feature. If blob files are already present, it will be directly loaded.\n"
       "\t    [OpenVINO only] [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU device(Reduces the CPU Utilization while using GPU) \n"
-      "\t    [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU enable_npu_fast_compile|true num_of_threads|5 enable_opencl_throttling|true cache_dir|\"<path>\"\"\n"
+      "\t    [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU num_of_threads|5 enable_opencl_throttling|true cache_dir|\"<path>\"\"\n"
       "\n"
       "\t    [QNN only] [backend_path]: QNN backend path. e.g '/folderpath/libQnnHtp.so', '/folderpath/libQnnCpu.so'.\n"
       "\t    [QNN only] [profiling_level]: QNN profiling level, options: 'basic', 'detailed', default 'off'.\n"
diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc
index fc1bdb10d7453..9286bce328592 100644
--- a/onnxruntime/test/perftest/ort_test_session.cc
+++ b/onnxruntime/test/perftest/ort_test_session.cc
@@ -754,13 +754,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
             ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n");
           }
         }
-      } else if (key == "enable_npu_fast_compile") {
-        if (value == "true" || value == "True" ||
-            value == "false" || value == "False") {
-          ov_options[key] = value;
-        } else {
-          ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n");
-        }
       } else if (key == "enable_opencl_throttling") {
         if (value == "true" || value == "True" ||
             value == "false" || value == "False") {
@@ -802,17 +795,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
         } else {
           ov_options[key] = value;
         }
-      } else if (key == "export_ep_ctx_blob") {
-        if (value == "true" || value == "True" ||
-            value == "false" || value == "False") {
-          ov_options[key] = value;
-        } else {
-          ORT_THROW(
-              "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' "
-              "should be a boolean i.e. true or false. Default value is false.\n");
-        }
       } else {
-        ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n");
+        ORT_THROW(
+            "[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO."
+            "['device_type', 'precision', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes',"
+            "'enable_qdq_optimizer', 'model_priority' ] \n");
       }
     }
     session_options.AppendExecutionProvider_OpenVINO_V2(ov_options);