diff --git a/cmake/onnxruntime_providers_openvino.cmake b/cmake/onnxruntime_providers_openvino.cmake index 5d1a481d40abc..e559583fae8f5 100644 --- a/cmake/onnxruntime_providers_openvino.cmake +++ b/cmake/onnxruntime_providers_openvino.cmake @@ -36,6 +36,7 @@ onnxruntime_add_include_to_target(onnxruntime_providers_openvino onnxruntime_common onnx) install(FILES ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino/openvino_provider_factory.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/) + set_target_properties(onnxruntime_providers_openvino PROPERTIES CXX_STANDARD 20) set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime") if(NOT MSVC) diff --git a/include/onnxruntime/core/graph/graph.h b/include/onnxruntime/core/graph/graph.h index 994087d985235..eb9581e8018d1 100644 --- a/include/onnxruntime/core/graph/graph.h +++ b/include/onnxruntime/core/graph/graph.h @@ -571,6 +571,13 @@ class Node { gsl::span output_args, const NodeAttributes* attributes, std::string_view domain); + void Init(std::string_view name, + std::string_view op_type, + std::string_view description, + gsl::span input_args, + gsl::span output_args, + NodeAttributes&& attributes, + std::string_view domain); #endif #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) @@ -952,6 +959,13 @@ class Graph { // NOLINT(clang-analyzer-optin.performance.Padding): preserve exi const NodeAttributes* attributes = nullptr, const std::string& domain = kOnnxDomain); + Node& AddNode(const std::string& name, + const std::string& op_type, + const std::string& description, + gsl::span input_args, + gsl::span output_args, + NodeAttributes&& attributes, + const std::string& domain = kOnnxDomain); Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4674db42fb1c9..5a5b35a4261e4 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -631,7 +631,6 @@ typedef struct OrtMIGraphXProviderOptions { typedef struct OrtOpenVINOProviderOptions { #ifdef __cplusplus OrtOpenVINOProviderOptions() : device_type{}, - enable_npu_fast_compile{}, device_id{}, num_of_threads{}, cache_dir{}, @@ -644,7 +643,6 @@ typedef struct OrtOpenVINOProviderOptions { * Valid settings are one of: "CPU_FP32", "CPU_FP16", "GPU_FP32", "GPU_FP16" */ const char* device_type; - unsigned char enable_npu_fast_compile; ///< 0 = disabled, nonzero = enabled const char* device_id; size_t num_of_threads; ///< 0 = Use default number of threads const char* cache_dir; // path is set to empty by default diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 7e82edabedbb3..e8a5855b36496 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -915,6 +915,42 @@ void Node::Init(std::string_view name, } } } +void Node::Init(std::string_view name, + std::string_view op_type, + std::string_view description, + gsl::span input_args, + gsl::span output_args, + NodeAttributes&& attributes, + std::string_view domain) { + name_ = name; + op_type_ = op_type; + description_ = description; + definitions_.input_defs.assign(input_args.begin(), input_args.end()); + definitions_.output_defs.assign(output_args.begin(), output_args.end()); + domain_ = domain; + can_be_saved_ = true; + priority_ = 0; + if (kOnnxDomainAlias == domain_) { + domain_ = kOnnxDomain; + } + + // Set each arg count as 1 by default. + // It could be adjusted when resolving the node with its operator + // information. + definitions_.input_arg_count.assign(input_args.size(), 1); + + attributes_ = std::move(attributes); + + for (auto& name_to_attr : attributes_) { + if (utils::HasGraph(name_to_attr.second)) { +#if !defined(ORT_MINIMAL_BUILD) + CreateSubgraph(name_to_attr.first); +#else + ORT_THROW("Creating node with a subgraph via AddNode is not supported in this build."); +#endif + } + } +} #endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS) #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) @@ -3923,6 +3959,35 @@ Node& Graph::AddNode(const std::string& name, return *node; } +Node& Graph::AddNode(const std::string& name, + const std::string& op_type, + const std::string& description, + gsl::span input_args, + gsl::span output_args, + NodeAttributes&& attributes, + const std::string& domain) { + InlinedVector inputs; + InlinedVector outputs; + inputs.resize(input_args.size()); + outputs.resize(output_args.size()); + int i = 0; + for (auto input_arg : input_args) { + inputs[i++] = &GetOrCreateNodeArg(input_arg->Name(), input_arg->TypeAsProto()); + } + i = 0; + for (auto output_arg : output_args) { + outputs[i++] = &GetOrCreateNodeArg(output_arg->Name(), output_arg->TypeAsProto()); + } + + const gsl::not_null node = AllocateNode(); + node->Init(name, op_type, description, inputs, outputs, std::move(attributes), domain); + if (0 != op_type.compare(kNoOp)) { + GraphProtoSyncNeeded(true); + } + + return *node; +} + bool Graph::RemoveNode(NodeIndex p_index) { auto node = GetNode(p_index); if (nullptr == node) { diff --git a/onnxruntime/core/providers/openvino/backend_manager.cc b/onnxruntime/core/providers/openvino/backend_manager.cc index 18a6257910a56..4d2e38022b66f 100644 --- a/onnxruntime/core/providers/openvino/backend_manager.cc +++ b/onnxruntime/core/providers/openvino/backend_manager.cc @@ -155,32 +155,28 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie auto compiled_model = concrete_backend_->GetOVCompiledModel(); std::string graph_name = ""; // Epctx file path from SO is mapped to cache_dir variable for OVEP for readability - if (global_context_.cache_dir != "") { + if (!global_context_.cache_dir.empty()) { graph_name = global_context_.cache_dir; } else { graph_name = global_context_.onnx_model_path_name; // Remove extension so we can append suffix to form the complete name of output graph - graph_name = [&]() { - size_t dot = graph_name.find_last_of("."); - if (dot == std::string::npos) return graph_name; - return graph_name.substr(0, dot); - }(); - graph_name = graph_name + "_ctx.onnx"; + size_t dot = global_context_.onnx_model_path_name.find_last_of("."); + graph_name = graph_name.substr(0, dot); + if (dot != std::string::npos) graph_name += "_ctx.onnx"; } + // If embed_mode, then pass on the serialized blob // If not embed_mode, dump the blob here and only pass on the path to the blob if (global_context_.ep_context_embed_mode) { std::ostringstream model_blob_stream; compiled_model.export_model(model_blob_stream); - model_blob_str = model_blob_stream.str(); - ORT_ENFORCE(model_blob_str.size() != 0); + model_blob_str = std::move(model_blob_stream).str(); + if (model_blob_str.empty()) { + ORT_THROW("Model blob stream is empty after exporting the compiled model."); + } } else { // Remove extension so we can append suffix to form the complete name of output graph - auto blob_name = [&]() { - size_t dot = graph_name.find_last_of("."); - if (dot == std::string::npos) return graph_name; - return graph_name.substr(0, dot); - }(); + auto blob_name = graph_name.substr(0, graph_name.find_last_of(".")); std::ofstream blob_file(blob_name + ".blob", std::ios::out | std::ios::trunc | std::ios::binary); if (!blob_file) { @@ -194,7 +190,7 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie graph_name, logger, global_context_.ep_context_embed_mode, - model_blob_str, + std::move(model_blob_str), openvino_sdk_version_)); return Status::OK(); diff --git a/onnxruntime/core/providers/openvino/backends/basic_backend.cc b/onnxruntime/core/providers/openvino/backends/basic_backend.cc index d79aa35be6418..0ee2926ce1fcc 100644 --- a/onnxruntime/core/providers/openvino/backends/basic_backend.cc +++ b/onnxruntime/core/providers/openvino/backends/basic_backend.cc @@ -66,7 +66,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, exe_network_ = global_context_.ie_core.ImportModel(model_stream, remote_context_, subgraph_context_.subgraph_name); - ie_cnn_network_ = exe_network_.Get().get_runtime_model(); } else if ((global_context.device_type.find("GPU") != std::string::npos) && (global_context_.context != nullptr)) { LOGS_DEFAULT(INFO) << log_tag << "IO Buffering Enabled"; @@ -75,7 +74,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); exe_network_ = global_context_.ie_core.CompileModel( ie_cnn_network_, remote_context_, subgraph_context_.subgraph_name); - ie_cnn_network_ = exe_network_.Get().get_runtime_model(); } else { ie_cnn_network_ = CreateOVModel(model_proto, global_context_, subgraph_context_, const_outputs_map_); exe_network_ = global_context_.ie_core.CompileModel( @@ -91,7 +89,15 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, device_config, global_context_.ep_context_embed_mode, subgraph_context_.subgraph_name); - ie_cnn_network_ = exe_network_.Get().get_runtime_model(); + // ie_cnn_network_ = exe_network_.Get().get_runtime_model(); + } else if (global_context_.export_ep_ctx_blob && + hw_target.find("NPU") != std::string::npos) { + std::shared_ptr ov_model; + { + const std::string model = model_proto.SerializeAsString(); + ov_model = global_context_.ie_core.Get().read_model(model, ov::Tensor()); + } + exe_network_ = OVExeNetwork(global_context_.ie_core.Get().compile_model(ov_model, hw_target, device_config)); } else if ((!subgraph_context_.has_dynamic_input_shape) && ((hw_target.find("AUTO") == std::string::npos) || (global_context_.OpenVINO_Version.at(0) >= 2024 && global_context_.OpenVINO_Version.at(1) > 2))) { @@ -102,7 +108,6 @@ BasicBackend::BasicBackend(const ONNX_NAMESPACE::ModelProto& model_proto, hw_target, device_config, subgraph_context_.subgraph_name); - ie_cnn_network_ = exe_network_.Get().get_runtime_model(); } else { // For all other types use ov::Model Type ie_cnn_network_ = CreateOVModel(model_proto, global_context_, const_outputs_map_); exe_network_ = global_context_.ie_core.CompileModel( @@ -270,14 +275,14 @@ void BasicBackend::StartAsyncInference(Ort::KernelContext& context, OVInferReque input_tensor_shape[tensor_iter] = *i; tensor_iter += 1; } - auto input = ie_cnn_network_->get_parameters().at(input_idx); + auto input = graph_input_info.at(input_idx); OVTensorPtr tensor_ptr; // avoid input copies on the CPU device if (global_context_.device_type.find("CPU") != std::string::npos) { - tensor_ptr = std::make_shared(input->get_element_type(), input_tensor_shape, + tensor_ptr = std::make_shared(input.get_element_type(), input_tensor_shape, (void*)tensor_data); } else { - tensor_ptr = std::make_shared(input->get_element_type(), input_tensor_shape); + tensor_ptr = std::make_shared(input.get_element_type(), input_tensor_shape); FillInputBlob(tensor_ptr, batch_slice_idx, input_name, context, subgraph_context_); } @@ -341,9 +346,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe const void* tensor_data = tensor.GetTensorRawData(); const cl::Buffer* shared_buffer_const = static_cast(tensor_data); // Create an Input Remote Blob - auto input = ie_cnn_network_->get_parameters().at(0); + auto input = graph_input_info.at(0); auto remote_blob = remote_context_->create_tensor( - input->get_element_type(), input->get_shape(), *shared_buffer_const); + input.get_element_type(), input.get_shape(), *shared_buffer_const); ov::Tensor tensor_remote = static_cast(remote_blob); OVTensorPtr tensor_ptr = std::make_shared(tensor_remote); infer_request->SetTensor(input_name, tensor_ptr); @@ -392,9 +397,9 @@ void BasicBackend::StartRemoteAsyncInference(Ort::KernelContext& context, OVInfe const void* tensor_data = tensor.GetTensorRawData(); const cl::Buffer* shared_buffer_const = static_cast(tensor_data); // Create a shared Blob, set the Infer Request Output Blob - auto output = ie_cnn_network_->get_results().at(0); + auto output = graph_output_info.at(0); auto remote_tensor = - remote_context_->create_tensor(output->get_element_type(), output->get_shape(), *shared_buffer_const); + remote_context_->create_tensor(output.get_element_type(), output.get_shape(), *shared_buffer_const); ov::Tensor tensor_t = static_cast(remote_tensor); OVTensorPtr tensor_ptr = std::make_shared(tensor_t); try { diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 598e985676f8d..91f195ed9144f 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -15,7 +15,6 @@ namespace openvino_ep { struct GlobalContext { OVCore ie_core; bool is_wholly_supported_graph = false; - bool enable_npu_fast_compile = false; bool enable_opencl_throttling = false; bool disable_dynamic_shapes = false; bool ep_context_embed_mode = true; diff --git a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc index e2df9c83f15ae..ee9486a62ea37 100644 --- a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include "core/providers/openvino/onnx_ctx_model_helper.h" @@ -18,71 +19,76 @@ Status EPCtxHandler::ExportEPCtxModel(const GraphViewer& graph_viewer, const std::string& graph_name, const logging::Logger& logger, const bool& ep_context_embed_mode, - const std::string& model_blob_str, + std::string&& model_blob_str, const std::string& openvino_sdk_version) const { auto model_build = graph_viewer.CreateModel(logger); auto& graph_build = model_build->MainGraph(); // Get graph inputs and outputs - std::vector inputs, outputs; - for (auto input : graph_viewer.GetInputs()) { - auto& n_input = graph_build.GetOrCreateNodeArg(input->Name(), input->TypeAsProto()); - inputs.push_back(&n_input); - } - for (auto output : graph_viewer.GetOutputs()) { - auto& n_output = graph_build.GetOrCreateNodeArg(output->Name(), output->TypeAsProto()); - outputs.push_back(&n_output); - } + const auto& viewer_inputs = graph_viewer.GetInputs(); + const auto& viewer_outputs = graph_viewer.GetOutputs(); + std::vector inputs(viewer_inputs.size()), outputs(viewer_outputs.size()); + auto transform_f = [&](const onnxruntime::NodeArg* iter) { return &graph_build.GetOrCreateNodeArg(iter->Name(), iter->TypeAsProto()); }; + auto fill_vectors = [transform_f](auto& src, auto& dst) { + std::transform(src.begin(), src.end(), dst.begin(), transform_f); + }; + fill_vectors(viewer_inputs, inputs); + fill_vectors(viewer_outputs, outputs); // Create EP context node attributes - auto attr_0 = ONNX_NAMESPACE::AttributeProto::Create(); - auto attr_1 = ONNX_NAMESPACE::AttributeProto::Create(); - auto attr_2 = ONNX_NAMESPACE::AttributeProto::Create(); - auto attr_3 = ONNX_NAMESPACE::AttributeProto::Create(); - - // embed mode - attr_0->set_name(EMBED_MODE); - attr_0->set_type(onnx::AttributeProto_AttributeType_INT); - attr_0->set_i(ep_context_embed_mode); - // ep context - attr_1->set_name(EP_CACHE_CONTEXT); - attr_1->set_type(onnx::AttributeProto_AttributeType_STRING); - attr_1->set_s(model_blob_str); - // sdk version - attr_2->set_name(EP_SDK_VER); - attr_2->set_type(onnx::AttributeProto_AttributeType_STRING); - attr_2->set_s(openvino_sdk_version); - // source - attr_3->set_name(SOURCE); - attr_3->set_type(onnx::AttributeProto_AttributeType_STRING); - attr_3->set_s(kOpenVINOExecutionProvider); - auto node_attributes = ONNX_NAMESPACE::NodeAttributes::Create(); node_attributes->reserve(4); - node_attributes->emplace(EMBED_MODE, *attr_0); - node_attributes->emplace(EP_CACHE_CONTEXT, *attr_1); - node_attributes->emplace(EP_SDK_VER, *attr_2); - node_attributes->emplace(SOURCE, *attr_3); - + { + // Create EP context node attributes + + // embed mode + auto embed_mode_attr = ONNX_NAMESPACE::AttributeProto::Create(); + embed_mode_attr->set_name(EMBED_MODE); + embed_mode_attr->set_type(onnx::AttributeProto_AttributeType_INT); + embed_mode_attr->set_i(ep_context_embed_mode); + node_attributes->emplace(EMBED_MODE, std::move(*embed_mode_attr)); + + // ep context + auto ep_cache_context_attr = ONNX_NAMESPACE::AttributeProto::Create(); + ep_cache_context_attr->set_name(EP_CACHE_CONTEXT); + ep_cache_context_attr->set_type(onnx::AttributeProto_AttributeType_STRING); + ep_cache_context_attr->set_s(std::move(model_blob_str)); + node_attributes->emplace(EP_CACHE_CONTEXT, std::move(*ep_cache_context_attr)); + + // sdk version + auto sdk_version_attr = ONNX_NAMESPACE::AttributeProto::Create(); + sdk_version_attr->set_name(EP_SDK_VER); + sdk_version_attr->set_type(onnx::AttributeProto_AttributeType_STRING); + sdk_version_attr->set_s(openvino_sdk_version); + node_attributes->emplace(EP_SDK_VER, std::move(*sdk_version_attr)); + + // source + auto source_attr = ONNX_NAMESPACE::AttributeProto::Create(); + source_attr->set_name(SOURCE); + source_attr->set_type(onnx::AttributeProto_AttributeType_STRING); + source_attr->set_s(kOpenVINOExecutionProvider); + node_attributes->emplace(SOURCE, std::move(*source_attr)); + } // Create EP context node - graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, node_attributes.get(), kMSDomain); + graph_build.AddNode(graph_name, EPCONTEXT_OP, "", inputs, outputs, std::move(*node_attributes), kMSDomain); ORT_ENFORCE(graph_build.Resolve().IsOK()); - // Serialize modelproto to string - auto new_graph_viewer = graph_build.CreateGraphViewer(); - auto model = new_graph_viewer->CreateModel(logger); - auto model_proto = model->ToProto(); - new_graph_viewer->ToProto(*model_proto->mutable_graph(), true, true); - model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); - - // Finally, dump the model - std::ofstream epctx_onnx_model(graph_name, - std::ios::out | std::ios::trunc | std::ios::binary); - if (!epctx_onnx_model) { - ORT_THROW("Unable to create epctx onnx model file "); - } - model_proto->SerializeToOstream(epctx_onnx_model); + { + // Serialize modelproto to string + auto model_proto = model_build->ToProto(); + model_proto->set_ir_version(ONNX_NAMESPACE::Version::IR_VERSION); + + // Finally, dump the model + std::ofstream epctx_onnx_model(graph_name, + std::ios::out | std::ios::trunc | std::ios::binary); + if (!epctx_onnx_model) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unable to create epctx onnx model file"); + } + if (!model_proto->SerializeToOstream(epctx_onnx_model)) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to serialize model to file"); + } + } LOGS_DEFAULT(VERBOSE) << "[OpenVINO EP] Export blob as EPContext Node"; return Status::OK(); diff --git a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h index 610e9fd49c901..c631d011d02b1 100644 --- a/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h +++ b/onnxruntime/core/providers/openvino/onnx_ctx_model_helper.h @@ -28,7 +28,7 @@ class EPCtxHandler { const std::string& graph_name, const logging::Logger& logger, const bool& ep_context_embed_mode, - const std::string& model_blob_str, + std::string&& model_blob_str, const std::string& openvino_sdk_version) const; Status ImportBlobFromEPCtxModel(const GraphViewer& graph_viewer); bool CheckForOVEPCtxNode(const GraphViewer& graph_viewer, std::string openvino_sdk_version) const; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc index 29c45916795d3..9f7775e05ad7d 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.cc +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.cc @@ -22,7 +22,6 @@ OpenVINOExecutionProvider::OpenVINOExecutionProvider(const OpenVINOExecutionProv global_context_ = std::make_unique(); global_context_->device_type = info.device_type_; global_context_->precision_str = info.precision_; - global_context_->enable_npu_fast_compile = info.enable_npu_fast_compile_; global_context_->cache_dir = info.cache_dir_; global_context_->model_priority = info.model_priority_; global_context_->num_streams = info.num_streams_; diff --git a/onnxruntime/core/providers/openvino/openvino_execution_provider.h b/onnxruntime/core/providers/openvino/openvino_execution_provider.h index 030e5bba71b67..e2f80511dcca1 100644 --- a/onnxruntime/core/providers/openvino/openvino_execution_provider.h +++ b/onnxruntime/core/providers/openvino/openvino_execution_provider.h @@ -79,7 +79,6 @@ static std::vector parseDevices(const std::string& device_string, struct OpenVINOExecutionProviderInfo { std::string device_type_{""}; std::string precision_{""}; - bool enable_npu_fast_compile_{false}; size_t num_of_threads_{0}; std::string cache_dir_{""}; std::string model_priority_{""}; @@ -95,14 +94,13 @@ struct OpenVINOExecutionProviderInfo { OpenVINOExecutionProviderInfo() = delete; explicit OpenVINOExecutionProviderInfo(const std::string& dev_type, const std::string& precision, - bool enable_npu_fast_compile, size_t num_of_threads, + size_t num_of_threads, const std::string& cache_dir, const std::string& model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, bool export_ep_ctx_blob, bool enable_qdq_optimizer, bool disable_cpu_fallback, bool so_epctx_embed_mode) : precision_(std::move(precision)), - enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), cache_dir_(std::move(cache_dir)), model_priority_(std::move(model_priority)), diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 4a13071499368..8e209a5acea4f 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -9,7 +9,7 @@ namespace onnxruntime { struct OpenVINOProviderFactory : IExecutionProviderFactory { OpenVINOProviderFactory(const char* device_type, const char* precision, - bool enable_npu_fast_compile, size_t num_of_threads, + size_t num_of_threads, const char* cache_dir, const char* model_priority, int num_streams, void* context, bool enable_opencl_throttling, bool disable_dynamic_shapes, @@ -17,7 +17,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { bool disable_cpu_fallback, bool so_epctx_embed_mode) : precision_(precision), - enable_npu_fast_compile_(enable_npu_fast_compile), num_of_threads_(num_of_threads), model_priority_(model_priority), num_streams_(num_streams), @@ -30,10 +29,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { so_epctx_embed_mode_(so_epctx_embed_mode) { device_type_ = (device_type == nullptr) ? "" : device_type; cache_dir_ = (cache_dir == nullptr) ? "" : cache_dir; - if (cache_dir != nullptr) { - free(const_cast(static_cast(cache_dir))); - cache_dir = nullptr; - } } ~OpenVINOProviderFactory() override { @@ -44,7 +39,6 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { private: std::string device_type_; std::string precision_; - bool enable_npu_fast_compile_; size_t num_of_threads_; std::string cache_dir_; std::string model_priority_; @@ -59,7 +53,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { }; std::unique_ptr OpenVINOProviderFactory::CreateProvider() { - OpenVINOExecutionProviderInfo info(device_type_, precision_, enable_npu_fast_compile_, num_of_threads_, + OpenVINOExecutionProviderInfo info(device_type_, precision_, num_of_threads_, cache_dir_, model_priority_, num_streams_, context_, enable_opencl_throttling_, disable_dynamic_shapes_, export_ep_ctx_blob_, enable_qdq_optimizer_, disable_cpu_fallback_, @@ -90,11 +84,9 @@ struct OpenVINO_Provider : Provider { // Not setting precision will execute with optimized precision for // best inference latency. set Precision=ACCURACY for executing models // with input precision for best accuracy. - bool enable_npu_fast_compile = false; // [enable_npu_fast_compile]: Fast-compile may be optionally enabled to - // speeds up the model's compilation to NPU device specific format. int num_of_threads = 0; // [num_of_threads]: Overrides the accelerator default value of number of // threads with this value at runtime. - const char* cache_dir = nullptr; // [cache_dir]: specify the path to + std::string cache_dir = ""; // [cache_dir]: specify the path to // dump and load the blobs for the model caching/kernel caching (GPU) // feature. If blob files are already present, it will be directly loaded. const char* model_priority = "DEFAULT"; // High-level OpenVINO model priority hint @@ -106,16 +98,16 @@ struct OpenVINO_Provider : Provider { // with this value at runtime. bool enable_opencl_throttling = false; // [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU // device (Reduces CPU Utilization when using GPU) - bool export_ep_ctx_blob = false; // Whether to export the pre-compiled blob as an EPContext model. - - void* context = nullptr; - - bool enable_qdq_optimizer = false; + bool export_ep_ctx_blob = false; // [Internal] Its linked to session_options.ep_context_enable + // Whether to export the pre-compiled blob as an EPContext model. + bool so_epctx_embed_mode = true; // [Internal] Its linked to session_options.ep_context_filepath + void* context = nullptr; // Address for IO Buffer - bool disable_cpu_fallback = false; + bool enable_qdq_optimizer = false; // Enables QDQ pruning for efficient inference latency with NPU - bool so_epctx_embed_mode = true; + bool disable_cpu_fallback = false; // Disable automatic fallback from NPU to CPU during compilation failures + std::string bool_flag = ""; if (provider_options_map.find("device_type") != provider_options_map.end()) { device_type = provider_options_map.at("device_type").c_str(); @@ -186,7 +178,7 @@ struct OpenVINO_Provider : Provider { } if (provider_options_map.find("cache_dir") != provider_options_map.end()) { - cache_dir = provider_options_map.at("cache_dir").c_str(); + cache_dir = provider_options_map.at("cache_dir"); } if (provider_options_map.find("context") != provider_options_map.end()) { @@ -228,16 +220,6 @@ struct OpenVINO_Provider : Provider { << "Executing with num_streams=1"; } } - std::string bool_flag = ""; - if (provider_options_map.find("enable_npu_fast_compile") != provider_options_map.end()) { - bool_flag = provider_options_map.at("enable_npu_fast_compile"); - if (bool_flag == "true" || bool_flag == "True") - enable_npu_fast_compile = true; - else if (bool_flag == "false" || bool_flag == "False") - enable_npu_fast_compile = false; - bool_flag = ""; - } - if (provider_options_map.find("enable_opencl_throttling") != provider_options_map.end()) { bool_flag = provider_options_map.at("enable_opencl_throttling"); if (bool_flag == "true" || bool_flag == "True") @@ -275,6 +257,7 @@ struct OpenVINO_Provider : Provider { disable_dynamic_shapes = false; } } + bool_flag = ""; } if (provider_options_map.find("so_export_ep_ctx_blob") != provider_options_map.end()) { bool_flag = provider_options_map.at("so_export_ep_ctx_blob"); @@ -305,35 +288,28 @@ struct OpenVINO_Provider : Provider { if (provider_options_map.find("so_epctx_path") != provider_options_map.end()) { // The path to dump epctx model is valid only when epctx is enabled. // Overrides the cache_dir option to dump model cache files from OV. - if (export_ep_ctx_blob) { - auto ep_context_file_path_ = provider_options_map.at("so_epctx_path"); - auto file_path = std::filesystem::path(ep_context_file_path_); + if (export_ep_ctx_blob && + !provider_options_map.at("so_epctx_path").empty()) { + cache_dir = provider_options_map.at("so_epctx_path"); + auto file_path = std::filesystem::path(cache_dir); // ep_context_file_path_ file extension must be .onnx - if (!ep_context_file_path_.empty()) { - if (file_path.extension().generic_string() == ".onnx") { - // ep_context_file_path_ must be provided as a directory, create it if doesn't exist - auto parent_path = file_path.parent_path(); - if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && - !std::filesystem::create_directory(parent_path)) { - ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n"); - } -#ifdef _WIN32 - cache_dir = _strdup(ep_context_file_path_.c_str()); -#else - cache_dir = strdup(ep_context_file_path_.c_str()); -#endif - } else { - ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + ep_context_file_path_ + " \n"); + if (file_path.extension().generic_string() == ".onnx") { + // ep_context_file_path_ must be provided as a directory, create it if doesn't exist + auto parent_path = file_path.parent_path(); + if (!parent_path.empty() && !std::filesystem::is_directory(parent_path) && + !std::filesystem::create_directory(parent_path)) { + ORT_THROW("[ERROR] [OpenVINO] Failed to create directory : " + file_path.parent_path().generic_string() + " \n"); } + } else { + ORT_THROW("[ERROR] [OpenVINO] Invalid ep_ctx_file_path" + cache_dir + " \n"); } } } return std::make_shared(const_cast(device_type.c_str()), const_cast(precision.c_str()), - enable_npu_fast_compile, num_of_threads, - cache_dir, + const_cast(cache_dir.c_str()), model_priority, num_streams, context, diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index 4527c0a89303c..66df8abb75b1f 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -379,6 +379,7 @@ struct ProviderHost { virtual float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) = 0; virtual const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) = 0; virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) = 0; + virtual void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) = 0; virtual void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) = 0; virtual void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) = 0; virtual void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0; @@ -849,6 +850,7 @@ struct ProviderHost { virtual std::unique_ptr NodeAttributes__find(const NodeAttributes* p, const std::string& key) = 0; virtual void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) = 0; virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0; + virtual void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) = 0; virtual void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) = 0; virtual void NodeAttributes__reserve(NodeAttributes* p, size_t size) = 0; @@ -900,6 +902,7 @@ struct ProviderHost { virtual Status Graph__Resolve(Graph* p) = 0; virtual void Graph__AddInitializedTensor(Graph* p, const ONNX_NAMESPACE::TensorProto& tensor) = 0; virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span& input_args, const gsl::span& output_args, const NodeAttributes* attributes, const std::string& domain) = 0; + virtual Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span& input_args, const gsl::span& output_args, NodeAttributes&& attributes, const std::string& domain) = 0; virtual Node& Graph__AddNode(Graph* p, const Node& other) = 0; virtual const std::vector& Graph__GetOutputs(const Graph* p) noexcept = 0; diff --git a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h index d98d91759b164..5545f7b08ac42 100644 --- a/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h +++ b/onnxruntime/core/providers/shared_library/provider_wrappedtypes.h @@ -114,6 +114,7 @@ struct AttributeProto final { float f() const { return g_host->AttributeProto__f(this); } const ONNX_NAMESPACE::TensorProto& t() const { return g_host->AttributeProto__t(this); } void set_s(const ::std::string& value) { return g_host->AttributeProto__set_s(this, value); } + void set_s(::std::string&& value) { return g_host->AttributeProto__set_s(this, ::std::move(value)); } void set_f(const float& value) { return g_host->AttributeProto__set_f(this, value); } void set_i(int64_t value) { return g_host->AttributeProto__set_i(this, value); } void set_t(const TensorProto& value) { return g_host->AttributeProto__set_t(this, value); } @@ -862,6 +863,7 @@ struct NodeAttributes final { IteratorHolder> find(const std::string& key) const { return g_host->NodeAttributes__find(this, key); } void insert(const NodeAttributes& v) { return g_host->NodeAttributes__insert(this, v); } void emplace(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__emplace(this, k, v); } + void emplace(const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) { g_host->NodeAttributes__emplace(this, k, std::move(v)); } void insert_or_assign(const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) { g_host->NodeAttributes__insert_or_assign(this, k, v); } void reserve(size_t size) { g_host->NodeAttributes__reserve(this, size); } @@ -947,6 +949,7 @@ struct Graph final { Status Resolve() { return g_host->Graph__Resolve(this); } void AddInitializedTensor(const ONNX_NAMESPACE::TensorProto& tensor) { return g_host->Graph__AddInitializedTensor(this, tensor); } Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span input_args, gsl::span output_args, const NodeAttributes* attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, attributes, domain); } + Node& AddNode(const std::string& name, const std::string& op_type, const std::string& description, gsl::span input_args, gsl::span output_args, NodeAttributes&& attributes, const std::string& domain) { return g_host->Graph__AddNode(this, name, op_type, description, input_args, output_args, std::move(attributes), domain); } Node& AddNode(const Node& other) { return g_host->Graph__AddNode(this, other); } const std::vector& GetOutputs() const noexcept { return g_host->Graph__GetOutputs(this); } diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index ff841950b4384..5c673e24eeee5 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -487,6 +487,7 @@ struct ProviderHostImpl : ProviderHost { float AttributeProto__f(const ONNX_NAMESPACE::AttributeProto* p) override { return p->f(); } const ONNX_NAMESPACE::TensorProto& AttributeProto__t(const ONNX_NAMESPACE::AttributeProto* p) override { return p->t(); } void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, const ::std::string& value) override { return p->set_s(value); } + void AttributeProto__set_s(ONNX_NAMESPACE::AttributeProto* p, ::std::string&& value) override { return p->set_s(::std::move(value)); } void AttributeProto__set_f(ONNX_NAMESPACE::AttributeProto* p, const float& value) override { return p->set_f(value); } void AttributeProto__set_i(ONNX_NAMESPACE::AttributeProto* p, int64_t value) override { return p->set_i(value); } void AttributeProto__set_t(ONNX_NAMESPACE::AttributeProto* p, const ONNX_NAMESPACE::TensorProto& value) override { *p->mutable_t() = value; } @@ -1112,6 +1113,7 @@ struct ProviderHostImpl : ProviderHost { } void NodeAttributes__insert(NodeAttributes* p, const NodeAttributes& v) override { return p->insert(v.begin(), v.end()); } void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->emplace(k, v); } + void NodeAttributes__emplace(NodeAttributes* p, const std::string& k, ONNX_NAMESPACE::AttributeProto&& v) override { p->emplace(k, std::move(v)); } void NodeAttributes__insert_or_assign(NodeAttributes* p, const std::string& k, const ONNX_NAMESPACE::AttributeProto& v) override { p->insert_or_assign(k, v); } void NodeAttributes__reserve(NodeAttributes* p, size_t size) override { p->reserve(size); } @@ -1189,6 +1191,9 @@ struct ProviderHostImpl : ProviderHost { Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span& input_args, const gsl::span& output_args, const NodeAttributes* attributes, const std::string& domain) override { return p->AddNode(name, op_type, description, input_args, output_args, attributes, domain); } + Node& Graph__AddNode(Graph* p, const std::string& name, const std::string& op_type, const std::string& description, const gsl::span& input_args, const gsl::span& output_args, NodeAttributes&& attributes, const std::string& domain) override { + return p->AddNode(name, op_type, description, input_args, output_args, ::std::move(attributes), domain); + } Node& Graph__AddNode(Graph* p, const Node& other) override { return p->AddNode(other); } @@ -1888,12 +1893,6 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O if (legacy_ov_options->device_type != nullptr) ov_options_converted_map["device_type"] = legacy_ov_options->device_type; - if (legacy_ov_options->enable_npu_fast_compile) { - ov_options_converted_map["enable_npu_fast_compile"] = "false"; - } else { - ov_options_converted_map["enable_npu_fast_compile"] = "true"; - } - if (legacy_ov_options->num_of_threads != '\0') ov_options_converted_map["num_of_threads"] = std::to_string(legacy_ov_options->num_of_threads); @@ -1916,7 +1915,6 @@ ProviderOptions OrtOpenVINOProviderOptionsToOrtOpenVINOProviderOptionsV2(const O // Add new provider option below ov_options_converted_map["num_streams"] = "1"; - ov_options_converted_map["export_ep_ctx_blob"] = "false"; ov_options_converted_map["model_priority"] = "DEFAULT"; ov_options_converted_map["enable_qdq_optimizer"] = "false"; return ov_options_converted_map; diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 47b8d75f22aea..311009157a69c 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -1045,12 +1045,6 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.first == "precision") { OV_provider_options_map[option.first] = option.second; continue; - } else if (option.first == "enable_npu_fast_compile") { - if (!(option.second == "True" || option.second == "true" || - option.second == "False" || option.second == "false")) { - ORT_THROW("Invalid value passed for enable_npu_fast_compile: ", option.second); - } - OV_provider_options_map[option.first] = option.second; } else if (option.first == "enable_opencl_throttling") { if (!(option.second == "True" || option.second == "true" || option.second == "False" || option.second == "false")) { @@ -1092,9 +1086,6 @@ std::unique_ptr CreateExecutionProviderInstance( } else if (option.first == "context") { OV_provider_options_map[option.first] = option.second; continue; - } else if (option.first == "export_ep_ctx_blob") { - OV_provider_options_map[option.first] = option.second; - continue; } else if (option.first == "enable_qdq_optimizer") { OV_provider_options_map[option.first] = option.second; continue; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 84c3bc16346f3..801767d38323c 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -74,11 +74,10 @@ namespace perftest { "\n" "\t [OpenVINO only] [device_type]: Overrides the accelerator hardware type and precision with these values at runtime.\n" "\t [OpenVINO only] [device_id]: Selects a particular hardware device for inference.\n" - "\t [OpenVINO only] [enable_npu_fast_compile]: Optionally enabled to speeds up the model's compilation on NPU device targets.\n" "\t [OpenVINO only] [num_of_threads]: Overrides the accelerator hardware type and precision with these values at runtime.\n" "\t [OpenVINO only] [cache_dir]: Explicitly specify the path to dump and load the blobs(Model caching) or cl_cache (Kernel Caching) files feature. If blob files are already present, it will be directly loaded.\n" "\t [OpenVINO only] [enable_opencl_throttling]: Enables OpenCL queue throttling for GPU device(Reduces the CPU Utilization while using GPU) \n" - "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU enable_npu_fast_compile|true num_of_threads|5 enable_opencl_throttling|true cache_dir|\"\"\"\n" + "\t [Example] [For OpenVINO EP] -e openvino -i \"device_type|CPU num_of_threads|5 enable_opencl_throttling|true cache_dir|\"\"\"\n" "\n" "\t [QNN only] [backend_path]: QNN backend path. e.g '/folderpath/libQnnHtp.so', '/folderpath/libQnnCpu.so'.\n" "\t [QNN only] [profiling_level]: QNN profiling level, options: 'basic', 'detailed', default 'off'.\n" diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index fc1bdb10d7453..9286bce328592 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -754,13 +754,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); ORT_THROW("[ERROR] [OpenVINO] Unsupported inference precision is selected. CPU only supports FP32 . \n"); } } - } else if (key == "enable_npu_fast_compile") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW("[ERROR] [OpenVINO] The value for the key 'enable_npu_fast_compile' should be a boolean i.e. true or false. Default value is false.\n"); - } } else if (key == "enable_opencl_throttling") { if (value == "true" || value == "True" || value == "false" || value == "False") { @@ -802,17 +795,11 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); } else { ov_options[key] = value; } - } else if (key == "export_ep_ctx_blob") { - if (value == "true" || value == "True" || - value == "false" || value == "False") { - ov_options[key] = value; - } else { - ORT_THROW( - "[ERROR] [OpenVINO] The value for the key 'export_ep_ctx_blob' " - "should be a boolean i.e. true or false. Default value is false.\n"); - } } else { - ORT_THROW("[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO. ['device_type', 'device_id', 'enable_npu_fast_compile', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'] \n"); + ORT_THROW( + "[ERROR] [OpenVINO] wrong key type entered. Choose from the following runtime key options that are available for OpenVINO." + "['device_type', 'precision', 'num_of_threads', 'cache_dir', 'num_streams', 'enable_opencl_throttling', 'disable_dynamic_shapes'," + "'enable_qdq_optimizer', 'model_priority' ] \n"); } } session_options.AppendExecutionProvider_OpenVINO_V2(ov_options); diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino index 4382e12a1cd6c..5f525c1310412 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino @@ -1,7 +1,7 @@ ARG UBUNTU_VERSION=22.04 FROM ubuntu:${UBUNTU_VERSION} -ARG OPENVINO_VERSION=2024.0.0 +ARG OPENVINO_VERSION=2024.3.0 ARG PYTHON_VERSION=3.10 ADD scripts /tmp/scripts @@ -19,9 +19,9 @@ ENV IE_PLUGINS_PATH $INTEL_OPENVINO_DIR/runtime/lib/intel64 ENV DEBIAN_FRONTEND=noninteractive RUN cd /opt && mkdir -p intel && cd intel && \ - wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.0/linux/l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \ - tar xzf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64.tgz && \ - mv l_openvino_toolkit_ubuntu22_2024.0.0.14509.34caeefd078_x86_64 openvino_2024.0.0 && \ + wget https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \ + tar xzf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && rm -rf l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz && \ + mv l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64 openvino_2024.3.0 && \ cd $INTEL_OPENVINO_DIR/install_dependencies && ./install_openvino_dependencies.sh -y WORKDIR /root