From f7d1cad439c70f6cc17612f8cdff42942996e594 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Thu, 6 Mar 2025 22:46:53 -0800 Subject: [PATCH 01/13] Improve Qnn EP weight sharing feature to make all generated ctx.onnx model point to the same ctx.bin during generation to avoid the post-processing work --- .../qnn/builder/onnx_ctx_model_helper.cc | 40 +++++++++-- .../qnn/builder/onnx_ctx_model_helper.h | 4 +- .../providers/qnn/qnn_execution_provider.cc | 8 ++- .../core/providers/qnn/shared_context.h | 18 +++++ .../test/ep_weight_sharing_ctx_gen/main.cc | 70 +++++++------------ 5 files changed, 86 insertions(+), 54 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc index d85277627a3de..93b2acb5b002c 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.cc @@ -10,6 +10,7 @@ #include "core/providers/qnn/ort_api.h" #include "core/providers/qnn/builder/qnn_utils.h" #include "core/providers/qnn/builder/qnn_model.h" +#include "core/providers/qnn/shared_context.h" namespace onnxruntime { namespace qnn { @@ -207,7 +208,9 @@ Status CreateEPContextNodes(Model* model, const onnxruntime::PathString& context_model_path, bool qnn_context_embed_mode, uint64_t max_spill_fill_buffer_size, - const logging::Logger& logger) { + const logging::Logger& logger, + bool share_ep_contexts, + bool stop_share_ep_contexts) { auto& graph = model->MainGraph(); using namespace ONNX_NAMESPACE; @@ -241,6 +244,7 @@ Status CreateEPContextNodes(Model* model, ep_node.AddAttribute(EP_CACHE_CONTEXT, cache_payload); } else { onnxruntime::PathString context_bin_path; + std::string context_cache_name; auto pos = context_model_path.find_last_of(ORT_TSTR(".")); if (pos != std::string::npos) { context_bin_path = context_model_path.substr(0, pos); @@ -253,14 +257,36 @@ Status CreateEPContextNodes(Model* model, graph_name_in_file.replace(name_pos, strlen(kQnnExecutionProvider), ""); } context_bin_path = context_bin_path + ToPathString(graph_name_in_file + ".bin"); - std::string context_cache_name(std::filesystem::path(context_bin_path).filename().string()); - std::ofstream of_stream(context_bin_path.c_str(), std::ofstream::binary); - if (!of_stream) { - LOGS(logger, ERROR) << "Failed to open create context file."; - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to open context cache file."); + context_cache_name = std::filesystem::path(context_bin_path).filename().string(); + + // If generate ctx.onnx with share_ep_context enabled, all ctx.onnx should point to the same ctx.bin + if (share_ep_contexts) { + auto shared_ctx_bin_name = SharedContext::GetInstance().GetSharedCtxBinFileName(); + if (shared_ctx_bin_name.empty()) { + SharedContext::GetInstance().SetSharedCtxBinFileName(context_cache_name); + } else { + context_cache_name = shared_ctx_bin_name; + auto model_folder_path = std::filesystem::path(context_bin_path).parent_path().string(); + context_bin_path = ToPathString(model_folder_path + "/" + context_cache_name); + } + } + + // Write the ctx.bin file for the case: 1. no share_ep_context enabled, write for every session + // 2. share_ep_context enabled, only write for the last session which has stop_share_ep_contexts enabled + if (!share_ep_contexts || (share_ep_contexts && stop_share_ep_contexts)) { + std::ofstream of_stream(context_bin_path.c_str(), std::ofstream::binary); + if (!of_stream) { + LOGS(logger, ERROR) << "Failed to open create context file."; + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to open context cache file."); + } + of_stream.write(reinterpret_cast(buffer), buffer_size); } - of_stream.write(reinterpret_cast(buffer), buffer_size); + ep_node.AddAttribute(EP_CACHE_CONTEXT, context_cache_name); + if (share_ep_contexts && stop_share_ep_contexts) { + SharedContext::GetInstance().ResetSharedCtxBinFileName(); + } + ep_node.AddAttribute(MAX_SIZE, static_cast(max_spill_fill_buffer_size)); } } else { diff --git a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h index c54cd3ca6e90c..b037d5c3d2336 100644 --- a/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h +++ b/onnxruntime/core/providers/qnn/builder/onnx_ctx_model_helper.h @@ -65,6 +65,8 @@ Status CreateEPContextNodes(Model* model, const onnxruntime::PathString& context_model_path, bool qnn_context_embed_mode, uint64_t max_spill_fill_buffer_size, - const logging::Logger& logger); + const logging::Logger& logger, + bool share_ep_contexts, + bool stop_share_ep_contexts); } // namespace qnn } // namespace onnxruntime diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index a5813dc2a4adc..ab6aa8b2ca06d 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -352,6 +352,10 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio LOGS_DEFAULT(VERBOSE) << "User specified enable_htp_weight_sharing: " << enable_htp_weight_sharing; } + if (qnn_context_embed_mode_ && enable_htp_weight_sharing) { + LOGS_DEFAULT(ERROR) << "[EP context generation:] Weight sharing enabled conflict with EP context embed mode. Inference will not work as expected!"; + } + // Add this option because this feature requires QnnSystem lib and it's no supported for Windows x86_64 platform enable_spill_fill_buffer_ = ParseBoolOption("enable_htp_spill_fill_buffer", false, provider_options_map); @@ -1051,7 +1055,9 @@ Status QNNExecutionProvider::Compile(const std::vector& fused context_model_path, qnn_context_embed_mode_, max_spill_fill_buffer_size, - logger)); + logger, + share_ep_contexts_, + stop_share_ep_contexts_)); if (share_ep_contexts_ && !stop_share_ep_contexts_ && nullptr == SharedContext::GetInstance().GetSharedQnnBackendManager()) { diff --git a/onnxruntime/core/providers/qnn/shared_context.h b/onnxruntime/core/providers/qnn/shared_context.h index 277a484ad8528..1402dc30fd37a 100644 --- a/onnxruntime/core/providers/qnn/shared_context.h +++ b/onnxruntime/core/providers/qnn/shared_context.h @@ -84,6 +84,21 @@ class SharedContext { qnn_backend_manager_.reset(); } + void SetSharedCtxBinFileName(std::string& shared_ctx_bin_file_name) { + const std::lock_guard lock(mtx_); + shared_ctx_bin_file_name_ = shared_ctx_bin_file_name; + } + + const std::string& GetSharedCtxBinFileName() { + const std::lock_guard lock(mtx_); + return shared_ctx_bin_file_name_; + } + + void ResetSharedCtxBinFileName() { + const std::lock_guard lock(mtx_); + shared_ctx_bin_file_name_.clear(); + } + private: SharedContext() = default; ~SharedContext() = default; @@ -94,6 +109,9 @@ class SharedContext { std::vector> shared_qnn_models_; // Used for compiling multiple models into same QNN context binary std::shared_ptr qnn_backend_manager_; + // Track the shared ctx binary .bin file name, all _ctx.onnx point to this .bin file + // only the last session generate the .bin file since it contains all graphs from all sessions. + std::string shared_ctx_bin_file_name_; // Producer sessions can be in parallel // Consumer sessions have to be after producer sessions initialized std::mutex mtx_; diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index 104cdbdfd5abc..2b0043fe30be8 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -16,10 +16,8 @@ using namespace onnxruntime; using ProviderOptions = std::unordered_map; // from the last context cache Onnx model, find the EPContext node with main_context=1, -// and get the QNN context binary file name, this context binary contains all graphs from all Onnx models // get the max spill fill buffer size static void GetLastContextBinaryFileName(const std::basic_string last_onnx_ctx_file, - std::string& last_ctx_bin_file, int64_t& max_size) { max_size = 0; @@ -37,9 +35,6 @@ static void GetLastContextBinaryFileName(const std::basic_string last if (attr.name() == "max_size") { max_size = attr.i(); } - if (attr.name() == "ep_cache_context") { - last_ctx_bin_file = attr.s(); - } } if (is_main_context) { return; @@ -50,11 +45,8 @@ static void GetLastContextBinaryFileName(const std::basic_string last onnx_file_stream.close(); } -// Update generated context cache Onnx model to make the main EPContext node point to -// the last QNN context binary file -// Remove not used QNN context binary file, only keep the last one which contains all graphs +// Update generated context cache Onnx model to have the same max_size (align with the last generated model) static void UpdateEpContextModel(const std::vector>& ep_ctx_files, - const std::string& last_qnn_ctx_binary_file_name, int64_t max_size) { for (auto ep_ctx_file : ep_ctx_files) { onnx::ModelProto model; @@ -65,9 +57,7 @@ static void UpdateEpContextModel(const std::vector> for (auto& node : *(model.mutable_graph()->mutable_node())) { if (node.op_type() == "EPContext") { int64_t is_main_context = 0; - std::string old_qnn_ctx_binary_file_name; int max_size_index = 0; - int ep_context_index = 0; for (auto i = 0; i < node.attribute_size(); ++i) { auto& attr = node.attribute()[i]; if (attr.name() == "main_context") { @@ -77,19 +67,9 @@ static void UpdateEpContextModel(const std::vector> max_size = attr.i(); max_size_index = i; } - if (attr.name() == "ep_cache_context") { - old_qnn_ctx_binary_file_name = attr.s(); - ep_context_index = 0; - } } if (is_main_context) { - auto path_str = ToPathString(ep_ctx_file); - auto path = std::filesystem::path(path_str); - auto file_path = path.replace_filename(old_qnn_ctx_binary_file_name); - std::remove(file_path.string().c_str()); - node.mutable_attribute(max_size_index)->set_i(max_size); - node.mutable_attribute(ep_context_index)->set_s(last_qnn_ctx_binary_file_name); } } } @@ -188,33 +168,33 @@ int real_main(int argc, char* argv[]) { } } - std::cout << "Start to update the generated Onnx model." << std::endl; - std::vector> ep_ctx_files; - ep_ctx_files.reserve(test_config.model_file_paths.size()); - for (auto model_path : test_config.model_file_paths) { - auto pos = model_path.find_last_of(ORT_TSTR(".")); - if (pos != std::string::npos) { - model_path = model_path.substr(0, pos) + ORT_TSTR("_ctx.onnx"); - } else { - model_path = model_path + ORT_TSTR("_ctx.onnx"); + // Update generated context cache Onnx model to have the same max_size (align with the last generated model) + // so that the inference session can be created with any order of the ctx.onnx models + // otherwise, user can also skip this step, but they need to create the inference session from the last generated ctx.onnx model + // since only the last ctx.onnx has EPContext nodes with correct max_size + const std::string enable_htp_weight_sharing = "enable_htp_spill_fill_buffer"; + if (provider_options.find(enable_htp_weight_sharing) == provider_options.end()) { + std::cout << "Start to update the generated Onnx model to reflect the max_size." << std::endl; + + // The steps below only used for spill fill buffer enabled + std::vector> ep_ctx_files; + ep_ctx_files.reserve(test_config.model_file_paths.size()); + for (auto model_path : test_config.model_file_paths) { + auto pos = model_path.find_last_of(ORT_TSTR(".")); + if (pos != std::string::npos) { + model_path = model_path.substr(0, pos) + ORT_TSTR("_ctx.onnx"); + } else { + model_path = model_path + ORT_TSTR("_ctx.onnx"); + } + ep_ctx_files.push_back(model_path); } - ep_ctx_files.push_back(model_path); - } - // Get the last context binary file name - std::string last_qnn_ctx_binary_file_name; - int64_t max_size = 0; - GetLastContextBinaryFileName(ep_ctx_files.back(), last_qnn_ctx_binary_file_name, max_size); - std::cout << "The last context binary file: " << last_qnn_ctx_binary_file_name << std::endl; - if (last_qnn_ctx_binary_file_name.empty()) { - throw Ort::Exception("Can't find QNN context binary file from the Onnx model.", OrtErrorCode::ORT_FAIL); - } - ep_ctx_files.pop_back(); + int64_t max_size = 0; + GetLastContextBinaryFileName(ep_ctx_files.back(), max_size); + ep_ctx_files.pop_back(); - // Update generated context cache Onnx model to make the main EPContext node point to - // the last QNN context binary file - // Remove not used QNN context binary file, only keep the last one only which contains all graphs - UpdateEpContextModel(ep_ctx_files, last_qnn_ctx_binary_file_name, max_size); + UpdateEpContextModel(ep_ctx_files, max_size); + } } ORT_CATCH(const Ort::Exception& e) { std::cerr << "Failed to generate context cache file: " << e.what(); From 71afe8e83ca7708d17d5d6a7a6c2a2f9126ff577 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 7 Mar 2025 09:18:17 -0800 Subject: [PATCH 02/13] update UT accordingly --- .../test/providers/qnn/qnn_ep_context_test.cc | 109 ++++++++---------- 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index 3dec74599abdf..e95a98818c37b 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1107,37 +1107,6 @@ static void DumpModelWithSharedCtx(const ProviderOptions& provider_options, Ort::Session session2(*ort_env, ToPathString(onnx_model_path2).c_str(), so); } -// Update generated context cache Onnx model to make the main EPContext node point to -// the last QNN context binary file -// Remove not used QNN context binary file, only keep the last one which contains all graphs -static void UpdateEpContextModel(const std::vector& ep_ctx_files, - const std::string& last_qnn_ctx_binary_file_name, - const Logger& logger) { - for (auto ep_ctx_file : ep_ctx_files) { - std::shared_ptr ctx_model; - auto path_str = ToPathString(ep_ctx_file); - ASSERT_STATUS_OK(Model::Load(path_str, ctx_model, nullptr, logger)); - auto& ctx_graph = ctx_model->MainGraph(); - GraphViewer graph_viewer(ctx_graph); - auto path = std::filesystem::path(path_str); - - for (auto& node : ctx_graph.Nodes()) { - if (node.OpType() == "EPContext") { - int64_t is_main_context = GetNodeAttr(node, "main_context", static_cast(0)); - if (1 == is_main_context) { - std::string old_qnn_ctx_binary_file_name = GetNodeAttr(node, "ep_cache_context", ""); - auto file_path = path.replace_filename(old_qnn_ctx_binary_file_name); - std::remove(file_path.string().c_str()); - node.ClearAttribute("ep_cache_context"); - node.AddAttribute("ep_cache_context", last_qnn_ctx_binary_file_name); - } - } - } - std::remove(ep_ctx_file.c_str()); - ASSERT_STATUS_OK(Model::Save(*ctx_model.get(), ToPathString(ep_ctx_file))); - } -} - static void GetModelInputNames(const std::string& model_path, std::vector& input_names, std::vector& output_names, @@ -1177,6 +1146,11 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { // Create QDQ models std::vector onnx_model_paths{"./weight_share1.onnx", "./weight_share2.onnx"}; + // cleanup in case some failure test doesn't remove them + for (auto model_path : onnx_model_paths) { + std::remove(model_path.c_str()); + } + std::vector ctx_model_paths; for (auto model_path : onnx_model_paths) { CreateQdqModel(model_path, DefaultLoggingManager().DefaultLogger()); @@ -1189,22 +1163,23 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { } ctx_model_paths.push_back(model_path); } + for (auto ctx_model_path : ctx_model_paths) { + std::remove(ctx_model_path.c_str()); + } DumpModelWithSharedCtx(provider_options, onnx_model_paths[0], onnx_model_paths[1]); - // Get the last context binary file name, the latest context binary file holds all graphs generated from all models - std::string last_qnn_ctx_binary_file_name; - GetContextBinaryFileName(ctx_model_paths.back(), last_qnn_ctx_binary_file_name, + std::string qnn_ctx_binary_file_name1; + GetContextBinaryFileName(ctx_model_paths[0], qnn_ctx_binary_file_name1, DefaultLoggingManager().DefaultLogger()); - EXPECT_TRUE(!last_qnn_ctx_binary_file_name.empty()); + EXPECT_TRUE(!qnn_ctx_binary_file_name1.empty()); - // Update generated context cache Onnx model to make the main EPContext node point to - // the last QNN context binary file - // Remove not used QNN context binary file, only keep the last one which contains all graphs - std::vector ctx_model_paths_to_update(ctx_model_paths); - ctx_model_paths_to_update.pop_back(); - UpdateEpContextModel(ctx_model_paths_to_update, last_qnn_ctx_binary_file_name, - DefaultLoggingManager().DefaultLogger()); + std::string qnn_ctx_binary_file_name2; + GetContextBinaryFileName(ctx_model_paths[1], qnn_ctx_binary_file_name2, + DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); + // 2 *_ctx.onn point to same .bin file + EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); Ort::SessionOptions so1; so1.SetLogId("so1"); @@ -1258,7 +1233,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { for (auto ctx_model_path : ctx_model_paths) { std::remove(ctx_model_path.c_str()); } - std::remove(last_qnn_ctx_binary_file_name.c_str()); + std::remove(qnn_ctx_binary_file_name1.c_str()); } // 1. Create 2 QDQ models @@ -1281,6 +1256,11 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions2) { // Create QDQ models std::vector onnx_model_paths{"./weight_share21.onnx", "./weight_share22.onnx"}; + // cleanup in case some failure test doesn't remove them + for (auto model_path : onnx_model_paths) { + std::remove(model_path.c_str()); + } + std::vector ctx_model_paths; for (auto model_path : onnx_model_paths) { CreateQdqModel(model_path, DefaultLoggingManager().DefaultLogger()); @@ -1293,24 +1273,23 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions2) { } ctx_model_paths.push_back(model_path); } + for (auto ctx_model_path : ctx_model_paths) { + std::remove(ctx_model_path.c_str()); + } DumpModelWithSharedCtx(provider_options, onnx_model_paths[0], onnx_model_paths[1]); - // Get the last context binary file name - std::string last_qnn_ctx_binary_file_name; - GetContextBinaryFileName(ctx_model_paths.back(), last_qnn_ctx_binary_file_name, + std::string qnn_ctx_binary_file_name1; + GetContextBinaryFileName(ctx_model_paths[0], qnn_ctx_binary_file_name1, + DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(!qnn_ctx_binary_file_name1.empty()); + + std::string qnn_ctx_binary_file_name2; + GetContextBinaryFileName(ctx_model_paths[1], qnn_ctx_binary_file_name2, DefaultLoggingManager().DefaultLogger()); - EXPECT_TRUE(!last_qnn_ctx_binary_file_name.empty()); - - // Update generated context cache Onnx model to make the main EPContext node point to - // the last QNN context binary file - // Remove not used QNN context binary file, only keep the last one which contains all graphs - std::vector ctx_model_paths_to_update(ctx_model_paths); - ctx_model_paths_to_update.pop_back(); - // The 2nd model still point to the context binary which includes all graphs - // The 1st model point to file not exists - UpdateEpContextModel(ctx_model_paths_to_update, "file_not_exist.bin", - DefaultLoggingManager().DefaultLogger()); + EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); + // 2 *_ctx.onn point to same .bin file + EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); Ort::SessionOptions so; so.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); @@ -1360,7 +1339,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions2) { for (auto ctx_model_path : ctx_model_paths) { std::remove(ctx_model_path.c_str()); } - std::remove(last_qnn_ctx_binary_file_name.c_str()); + std::remove(qnn_ctx_binary_file_name1.c_str()); } // For Ort sessions to generate the context binary, with session option ep.share_ep_contexts enabled @@ -1376,6 +1355,11 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { // Create QDQ models std::vector onnx_model_paths{"./weight_share1.onnx", "./weight_share2.onnx"}; + // cleanup in case some failure test doesn't remove them + for (auto model_path : onnx_model_paths) { + std::remove(model_path.c_str()); + } + std::vector ctx_model_paths; for (auto model_path : onnx_model_paths) { CreateQdqModel(model_path, DefaultLoggingManager().DefaultLogger()); @@ -1388,6 +1372,9 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { } ctx_model_paths.push_back(model_path); } + for (auto ctx_model_path : ctx_model_paths) { + std::remove(ctx_model_path.c_str()); + } Ort::SessionOptions so; so.AddConfigEntry(kOrtSessionOptionEpContextEnable, "1"); @@ -1411,9 +1398,8 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { DefaultLoggingManager().DefaultLogger()); EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); - auto file_size_1 = std::filesystem::file_size(qnn_ctx_binary_file_name1); - auto file_size_2 = std::filesystem::file_size(qnn_ctx_binary_file_name2); - EXPECT_TRUE(file_size_2 > file_size_1); + // 2 *_ctx.onn point to same .bin file + EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); // clean up for (auto model_path : onnx_model_paths) { @@ -1423,7 +1409,6 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { ASSERT_EQ(std::remove(ctx_model_path.c_str()), 0); } ASSERT_EQ(std::remove(qnn_ctx_binary_file_name1.c_str()), 0); - ASSERT_EQ(std::remove(qnn_ctx_binary_file_name2.c_str()), 0); } #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) From b4ce772c6769708fa31845b2d0711f460cc4688c Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 7 Mar 2025 09:30:41 -0800 Subject: [PATCH 03/13] format --- onnxruntime/test/providers/qnn/qnn_ep_context_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index e95a98818c37b..57cda2e23daff 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1399,7 +1399,7 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); // 2 *_ctx.onn point to same .bin file - EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); + EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); // clean up for (auto model_path : onnx_model_paths) { From bd2dfd0c543a5a2de37b19ddc05027b8e897e172 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 7 Mar 2025 13:45:53 -0800 Subject: [PATCH 04/13] update the tool --- onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index 2b0043fe30be8..ed4211fd6bfb8 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -168,12 +168,13 @@ int real_main(int argc, char* argv[]) { } } + // Only with enable_htp_spill_fill_buffer enabled: // Update generated context cache Onnx model to have the same max_size (align with the last generated model) // so that the inference session can be created with any order of the ctx.onnx models // otherwise, user can also skip this step, but they need to create the inference session from the last generated ctx.onnx model - // since only the last ctx.onnx has EPContext nodes with correct max_size + // since only the last ctx.onnx has EPContext nodes has the correct max_size covers graphs for all sessions const std::string enable_htp_weight_sharing = "enable_htp_spill_fill_buffer"; - if (provider_options.find(enable_htp_weight_sharing) == provider_options.end()) { + if (provider_options.find(enable_htp_weight_sharing) != provider_options.end()) { std::cout << "Start to update the generated Onnx model to reflect the max_size." << std::endl; // The steps below only used for spill fill buffer enabled From a3c2ab55a921c15e9ea11bdb41f3d457064068f1 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 7 Mar 2025 13:52:05 -0800 Subject: [PATCH 05/13] update the condition for enable_htp_spill_fill_buffer validation --- onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index ed4211fd6bfb8..e65321f99b707 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -174,7 +174,8 @@ int real_main(int argc, char* argv[]) { // otherwise, user can also skip this step, but they need to create the inference session from the last generated ctx.onnx model // since only the last ctx.onnx has EPContext nodes has the correct max_size covers graphs for all sessions const std::string enable_htp_weight_sharing = "enable_htp_spill_fill_buffer"; - if (provider_options.find(enable_htp_weight_sharing) != provider_options.end()) { + auto pos = provider_options.find(enable_htp_weight_sharing); + if (pos != provider_options.end() && pos->second == "1") { std::cout << "Start to update the generated Onnx model to reflect the max_size." << std::endl; // The steps below only used for spill fill buffer enabled From 1fe1d5afab9abe39be06583c36a28d1079099788 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Fri, 7 Mar 2025 13:55:47 -0800 Subject: [PATCH 06/13] update method name --- onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index e65321f99b707..6156ab32092a5 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -17,8 +17,8 @@ using ProviderOptions = std::unordered_map; // from the last context cache Onnx model, find the EPContext node with main_context=1, // get the max spill fill buffer size -static void GetLastContextBinaryFileName(const std::basic_string last_onnx_ctx_file, - int64_t& max_size) { +static void GetEpContextInfoFromLastContextModel(const std::basic_string last_onnx_ctx_file, + int64_t& max_size) { max_size = 0; onnx::ModelProto model; @@ -182,9 +182,9 @@ int real_main(int argc, char* argv[]) { std::vector> ep_ctx_files; ep_ctx_files.reserve(test_config.model_file_paths.size()); for (auto model_path : test_config.model_file_paths) { - auto pos = model_path.find_last_of(ORT_TSTR(".")); - if (pos != std::string::npos) { - model_path = model_path.substr(0, pos) + ORT_TSTR("_ctx.onnx"); + auto dot_pos = model_path.find_last_of(ORT_TSTR(".")); + if (dot_pos != std::string::npos) { + model_path = model_path.substr(0, dot_pos) + ORT_TSTR("_ctx.onnx"); } else { model_path = model_path + ORT_TSTR("_ctx.onnx"); } @@ -192,7 +192,7 @@ int real_main(int argc, char* argv[]) { } int64_t max_size = 0; - GetLastContextBinaryFileName(ep_ctx_files.back(), max_size); + GetEpContextInfoFromLastContextModel(ep_ctx_files.back(), max_size); ep_ctx_files.pop_back(); UpdateEpContextModel(ep_ctx_files, max_size); From b949fde928800f733993912c6ac3ab7d263bc676 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Mon, 10 Mar 2025 23:20:56 -0700 Subject: [PATCH 07/13] remove enable_htp_weight_sharing from provider option. it can be decided from session option ep.share_ep_contexts. It is enabled if ep.share_ep_contexts for the QDQ model. And it's for x64 only. --- .../core/session/onnxruntime_c_api.h | 3 --- .../qnn/builder/qnn_backend_manager.cc | 19 +++++++++++---- .../qnn/builder/qnn_backend_manager.h | 10 ++++---- .../providers/qnn/qnn_execution_provider.cc | 24 ++++--------------- .../command_args_parser.cc | 7 +++--- .../test/ep_weight_sharing_ctx_gen/main.cc | 9 ++----- 6 files changed, 29 insertions(+), 43 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 098de14bdfd61..9946d8e8fda45 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -3674,9 +3674,6 @@ struct OrtApi { * Enable the float32 model to be inferenced with fp16 precision. Otherwise, it will be fp32 precision. * - "0": With fp32 precision. * - "1": Default. With fp16 precision. - * "enable_htp_weight_sharing": Enable QNN weight sharing feature while compiling multiple graphs into one QNN context. - * - "0": Default. Disabled. - * - "1": Enabled. * "offload_graph_io_quantization": Offload graph input quantization and graph output dequantization to another * execution provider (typically CPU EP). * - "0": Disabled. QNN EP will handle quantization and dequantization of graph I/O. diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index 26d792c008edc..d8dd5901f9c00 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -538,7 +538,7 @@ Status SetQnnContextConfig(ContextPriority context_priority, QnnContext_Config_t return Status::OK(); } -Status QnnBackendManager::CreateContext() { +Status QnnBackendManager::CreateContext(bool enable_htp_weight_sharing) { if (true == context_created_) { LOGS_DEFAULT(INFO) << "Context created already."; return Status::OK(); @@ -547,7 +547,7 @@ Status QnnBackendManager::CreateContext() { QnnContext_Config_t context_config_weight_sharing = QNN_CONTEXT_CONFIG_INIT; QnnHtpContext_CustomConfig_t custom_config; custom_config.option = QNN_HTP_CONTEXT_CONFIG_OPTION_WEIGHT_SHARING_ENABLED; - custom_config.weightSharingEnabled = enable_htp_weight_sharing_; + custom_config.weightSharingEnabled = enable_htp_weight_sharing; context_config_weight_sharing.option = QNN_CONTEXT_CONFIG_OPTION_CUSTOM; context_config_weight_sharing.customConfig = &custom_config; @@ -810,7 +810,8 @@ Status QnnBackendManager::LoadCachedQnnContextFromBuffer(char* buffer, uint64_t // or generate Qnn context binary is enabled -- to get the max spill fill buffer size Status QnnBackendManager::SetupBackend(const logging::Logger& logger, bool load_from_cached_context, - bool need_load_system_lib) { + bool need_load_system_lib, + bool share_ep_contexts) { std::lock_guard lock(logger_recursive_mutex_); if (backend_setup_completed_) { LOGS(logger, VERBOSE) << "Backend setup already!"; @@ -865,9 +866,19 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, LOGS(logger, VERBOSE) << "InitializeProfiling succeed."; } + bool enable_htp_weight_sharing = false; + // weight sharing only available with offline generation on x64 platform, not available on the real device +#if defined(__aarch64__) || defined(_M_ARM64) + ORT_UNUSED_PARAMETER(share_ep_contexts); +#else + if (share_ep_contexts && !load_from_cached_context) { + enable_htp_weight_sharing = true; + } +#endif + if (!load_from_cached_context) { if (status.IsOK()) { - status = CreateContext(); + status = CreateContext(enable_htp_weight_sharing); } if (status.IsOK()) { LOGS(logger, VERBOSE) << "CreateContext succeed."; diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h index 3592af41f03df..bd451d9ba9c1d 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h @@ -43,7 +43,6 @@ struct QnnBackendManagerConfig { uint32_t device_id; QnnHtpDevice_Arch_t htp_arch; uint32_t soc_model; - bool enable_htp_weight_sharing; }; class QnnBackendManager : public std::enable_shared_from_this { @@ -67,8 +66,7 @@ class QnnBackendManager : public std::enable_shared_from_this qnn_saver_path_(config.qnn_saver_path), device_id_(config.device_id), htp_arch_(config.htp_arch), - soc_model_(config.soc_model), - enable_htp_weight_sharing_(config.enable_htp_weight_sharing) { + soc_model_(config.soc_model) { } ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(QnnBackendManager); @@ -84,7 +82,8 @@ class QnnBackendManager : public std::enable_shared_from_this // Initializes handles to QNN resources (device, logger, etc.). // NOTE: This function locks the internal `logger_recursive_mutex_`. - Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context, bool need_load_system_lib); + Status SetupBackend(const logging::Logger& logger, bool load_from_cached_context, + bool need_load_system_lib, bool share_ep_contexts); Status CreateHtpPowerCfgId(uint32_t deviceId, uint32_t coreId, uint32_t& htp_power_config_id); @@ -155,7 +154,7 @@ class QnnBackendManager : public std::enable_shared_from_this Status ReleaseProfilehandle(); - Status CreateContext(); + Status CreateContext(bool enable_htp_weight_sharing); Status ReleaseContext(); @@ -298,7 +297,6 @@ class QnnBackendManager : public std::enable_shared_from_this uint32_t device_id_ = 0; QnnHtpDevice_Arch_t htp_arch_ = QNN_HTP_DEVICE_ARCH_NONE; uint32_t soc_model_ = QNN_SOC_MODEL_UNKNOWN; - bool enable_htp_weight_sharing_ = false; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc index ab6aa8b2ca06d..e5b88a77b334c 100644 --- a/onnxruntime/core/providers/qnn/qnn_execution_provider.cc +++ b/onnxruntime/core/providers/qnn/qnn_execution_provider.cc @@ -337,22 +337,7 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio LOGS_DEFAULT(VERBOSE) << "User specified enable_htp_fp16_precision: " << enable_HTP_FP16_precision_; } - bool enable_htp_weight_sharing = false; - static const std::string QNN_HTP_WEIGHT_SHARING_ENABLED = "enable_htp_weight_sharing"; - auto htp_weight_sharing_enabled_pos = provider_options_map.find(QNN_HTP_WEIGHT_SHARING_ENABLED); - if (htp_weight_sharing_enabled_pos != provider_options_map.end()) { - if ("1" == htp_weight_sharing_enabled_pos->second) { - enable_htp_weight_sharing = true; - } else if ("0" == htp_weight_sharing_enabled_pos->second) { - enable_htp_weight_sharing = false; - } else { - LOGS_DEFAULT(VERBOSE) << "Invalid enable_htp_weight_sharing: " << enable_htp_weight_sharing - << " only 0 or 1 allowed. Set to 0."; - } - LOGS_DEFAULT(VERBOSE) << "User specified enable_htp_weight_sharing: " << enable_htp_weight_sharing; - } - - if (qnn_context_embed_mode_ && enable_htp_weight_sharing) { + if (qnn_context_embed_mode_ && share_ep_contexts_) { LOGS_DEFAULT(ERROR) << "[EP context generation:] Weight sharing enabled conflict with EP context embed mode. Inference will not work as expected!"; } @@ -410,8 +395,7 @@ QNNExecutionProvider::QNNExecutionProvider(const ProviderOptions& provider_optio qnn_saver_path, device_id_, htp_arch, - soc_model, - enable_htp_weight_sharing}); + soc_model}); } #if defined(_WIN32) @@ -705,7 +689,9 @@ QNNExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_viewer // It will load the QnnSystem lib if is_qnn_ctx_model=true, and // delay the Qnn context creation to Compile() using the cached context binary // or generate context cache enable, need to use use QnnSystem lib to parse the binary to get the max spill fill buffer size - auto rt = qnn_backend_manager_->SetupBackend(logger, is_qnn_ctx_model, context_cache_enabled_ && enable_spill_fill_buffer_); + auto rt = qnn_backend_manager_->SetupBackend(logger, is_qnn_ctx_model, + context_cache_enabled_ && enable_spill_fill_buffer_, + share_ep_contexts_); if (Status::OK() != rt) { LOGS(logger, ERROR) << "QNN SetupBackend failed " << rt.ErrorMessage(); return result; diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/command_args_parser.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/command_args_parser.cc index bf21d54ccde41..ccfccf2e08dfc 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/command_args_parser.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/command_args_parser.cc @@ -48,7 +48,6 @@ namespace qnnctxgen { "\t [QNN only] [htp_arch]: The minimum HTP architecture. The driver will use ops compatible with this architecture. eg: '0', '68', '69', '73', '75'. Defaults to '0' (none). \n" "\t [QNN only] [enable_htp_fp16_precision]: Enable the HTP_FP16 precision so that the float32 model will be inferenced with fp16 precision. \n" "\t Otherwise, it will be fp32 precision. Works for float32 model for HTP backend. Defaults to '1' (with FP16 precision.). \n" - "\t [QNN only] [enable_htp_weight_sharing]: Allows common weights across graphs to be shared and stored in a single context binary. Defaults to '1' (enabled).\n" "\t [QNN only] [offload_graph_io_quantization]: Offload graph input quantization and graph output dequantization to another EP (typically CPU EP). \n" "\t Defaults to '1' (QNN EP handles the graph I/O quantization and dequantization). \n" "\t [QNN only] [enable_htp_spill_fill_buffer]: Enable HTP spill file buffer, used while generating QNN context binary." @@ -161,8 +160,8 @@ static bool ParseSessionConfigs(const std::string& configs_string, std::string str = str_stream.str(); ORT_THROW("Wrong value for htp_graph_finalization_optimization_mode. select from: " + str); } - } else if (key == "enable_htp_fp16_precision" || key == "enable_htp_weight_sharing" || - key == "offload_graph_io_quantization" || key == "enable_htp_spill_fill_buffer") { + } else if (key == "enable_htp_fp16_precision" || key == "offload_graph_io_quantization" || + key == "enable_htp_spill_fill_buffer") { std::unordered_set supported_options = {"0", "1"}; if (supported_options.find(value) == supported_options.end()) { std::ostringstream str_stream; @@ -173,7 +172,7 @@ static bool ParseSessionConfigs(const std::string& configs_string, } } else { ORT_THROW(R"(Wrong key type entered. Choose from options: ['backend_path', 'vtcm_mb', 'htp_performance_mode', - 'htp_graph_finalization_optimization_mode', 'soc_model', 'htp_arch', 'enable_htp_fp16_precision', 'enable_htp_weight_sharing', + 'htp_graph_finalization_optimization_mode', 'soc_model', 'htp_arch', 'enable_htp_fp16_precision', 'offload_graph_io_quantization', 'enable_htp_spill_fill_buffer'])"); } diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index 6156ab32092a5..103559a141beb 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -144,11 +144,6 @@ int real_main(int argc, char* argv[]) { provider_options["backend_path"] = "libQnnHtp.so"; #endif - // set default QNN EP option to enable weight sharing if not set by user - const std::string enable_htp_weight_sharing = "enable_htp_weight_sharing"; - if (provider_options.find(enable_htp_weight_sharing) == provider_options.end()) { - provider_options[enable_htp_weight_sharing] = "1"; - } so.AppendExecutionProvider("QNN", provider_options); #else ORT_THROW("QNN is not supported in this build\n"); @@ -173,8 +168,8 @@ int real_main(int argc, char* argv[]) { // so that the inference session can be created with any order of the ctx.onnx models // otherwise, user can also skip this step, but they need to create the inference session from the last generated ctx.onnx model // since only the last ctx.onnx has EPContext nodes has the correct max_size covers graphs for all sessions - const std::string enable_htp_weight_sharing = "enable_htp_spill_fill_buffer"; - auto pos = provider_options.find(enable_htp_weight_sharing); + const std::string enable_htp_spill_fill_buffer = "enable_htp_spill_fill_buffer"; + auto pos = provider_options.find(enable_htp_spill_fill_buffer); if (pos != provider_options.end() && pos->second == "1") { std::cout << "Start to update the generated Onnx model to reflect the max_size." << std::endl; From b8899006eb208ee6d3e9b1befd56dc4a37a6b37f Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 09:51:18 -0700 Subject: [PATCH 08/13] fix UT by adding provider_options["soc_model"] = "60" since weight sharing is only available for v73 and higher --- onnxruntime/test/providers/qnn/qnn_ep_context_test.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index 57cda2e23daff..0de2b497db6ed 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1088,7 +1088,7 @@ static void CreateQdqModel(const std::string& model_file_name, const Logger& log ASSERT_STATUS_OK(onnxruntime::Model::Save(model, ToPathString(model_file_name))); } -static void DumpModelWithSharedCtx(const ProviderOptions& provider_options, +static void DumpModelWithSharedCtx(ProviderOptions provider_options, const std::string& onnx_model_path1, const std::string& onnx_model_path2) { Ort::SessionOptions so; @@ -1097,6 +1097,14 @@ static void DumpModelWithSharedCtx(const ProviderOptions& provider_options, // enable ep.share_ep_contexts so that QNNEP share the QnnBackendManager across sessions so.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); +#ifndef __aarch64__ +#ifndef _M_ARM64 + // weight sharing only available for v73 and higher + provider_options["soc_model"] = "60"; +#endif // !_M_ARM64 +#endif // !__aarch64__ + + so.AppendExecutionProvider("QNN", provider_options); // Create 2 sessions to generate context binary models, the 1st session will share the QnnBackendManager From 8570ada66f03062f10d434d388621b40a645428c Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 10:18:57 -0700 Subject: [PATCH 09/13] log warning if user want to enable weight sharing on device --- .../providers/qnn/builder/qnn_backend_manager.cc | 7 +++---- .../test/providers/qnn/qnn_ep_context_test.cc | 13 +------------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc index d8dd5901f9c00..0328f6c2014fa 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc @@ -867,14 +867,13 @@ Status QnnBackendManager::SetupBackend(const logging::Logger& logger, } bool enable_htp_weight_sharing = false; - // weight sharing only available with offline generation on x64 platform, not available on the real device + if (share_ep_contexts && !load_from_cached_context) { #if defined(__aarch64__) || defined(_M_ARM64) - ORT_UNUSED_PARAMETER(share_ep_contexts); + LOGS(logger, WARNING) << "Weight sharing only available with offline generation on x64 platform, not work on real device."; #else - if (share_ep_contexts && !load_from_cached_context) { enable_htp_weight_sharing = true; - } #endif + } if (!load_from_cached_context) { if (status.IsOK()) { diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index 0de2b497db6ed..c77ea624da3df 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1104,7 +1104,6 @@ static void DumpModelWithSharedCtx(ProviderOptions provider_options, #endif // !_M_ARM64 #endif // !__aarch64__ - so.AppendExecutionProvider("QNN", provider_options); // Create 2 sessions to generate context binary models, the 1st session will share the QnnBackendManager @@ -1384,23 +1383,13 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { std::remove(ctx_model_path.c_str()); } - Ort::SessionOptions so; - so.AddConfigEntry(kOrtSessionOptionEpContextEnable, "1"); - so.AddConfigEntry(kOrtSessionOptionEpContextEmbedMode, "0"); - // enable ep.share_ep_contexts so that QNNEP share the QnnBackendManager across sessions - so.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); - - so.AppendExecutionProvider("QNN", provider_options); + DumpModelWithSharedCtx(provider_options, onnx_model_paths[0], onnx_model_paths[1]); - Ort::Session session1(*ort_env, ToPathString(onnx_model_paths[0]).c_str(), so); std::string qnn_ctx_binary_file_name1; GetContextBinaryFileName(ctx_model_paths[0], qnn_ctx_binary_file_name1, DefaultLoggingManager().DefaultLogger()); EXPECT_TRUE(!qnn_ctx_binary_file_name1.empty()); - // Tell the EP stop share the QnnBackendManager from this session then on - so.AddConfigEntry(kOrtSessionOptionStopShareEpContexts, "1"); - Ort::Session session2(*ort_env, ToPathString(onnx_model_paths[1]).c_str(), so); std::string qnn_ctx_binary_file_name2; GetContextBinaryFileName(ctx_model_paths[1], qnn_ctx_binary_file_name2, DefaultLoggingManager().DefaultLogger()); From 51675d3b865d19d3b0a688dc36b0ac382d151924 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 11:30:17 -0700 Subject: [PATCH 10/13] update UT, remove duplicate test --- .../test/providers/qnn/qnn_ep_context_test.cc | 119 ++---------------- 1 file changed, 10 insertions(+), 109 deletions(-) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index c77ea624da3df..dfe5b88632cf0 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1138,11 +1138,10 @@ static void GetModelInputNames(const std::string& model_path, // 2. Initialize 2 Ort sessions which share the same QNN EP from these 2 QDQ models // with EpContextEnable = 1, to dump the context binary // so, the 2nd context binary contains the graph from the 1st model -// 3. Change the 1st context model to point to the 2nd context binary file -// 4. Start 2 ort session from the dumped context model, +// 3. Start 2 ort session from the dumped context model, // The 2nd session uses graph from 1st session -// 5. Run the 2nd session -TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { +// 4. Run the 2nd session +TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions) { ProviderOptions provider_options; #if defined(_WIN32) provider_options["backend_path"] = "QnnHtp.dll"; @@ -1187,7 +1186,11 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); // 2 *_ctx.onn point to same .bin file EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); + auto file_size_1 = std::filesystem::file_size(qnn_ctx_binary_file_name1); + EXPECT_TRUE(file_size_1 > 0); + // only load and run the session on real device +#if defined(__aarch64__) || defined(_M_ARM64) Ort::SessionOptions so1; so1.SetLogId("so1"); so1.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); @@ -1233,6 +1236,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { auto ort_outputs1 = session1.Run(Ort::RunOptions{}, input_names_c.data(), ort_inputs.data(), ort_inputs.size(), output_names_c.data(), 1); + #endif for (auto model_path : onnx_model_paths) { std::remove(model_path.c_str()); @@ -1243,111 +1247,6 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions1) { std::remove(qnn_ctx_binary_file_name1.c_str()); } -// 1. Create 2 QDQ models -// 2. Initialize 2 Ort sessions which share the same QNN EP from these 2 QDQ models -// with EpContextEnable = 1, to dump the context binary -// so, the 2nd context binary contains the graph from the 1st model -// 3. Change the 1st context model to point to a context binary file which is not exist -// 4. Start 2 ort session from the dumped context model, -// The 1st session uses the 2nd model, the 2nd session uses the 1st model -// so the 2nd session uses graph from the 1st session -// 6. Run the 2nd session -TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions2) { - ProviderOptions provider_options; -#if defined(_WIN32) - provider_options["backend_path"] = "QnnHtp.dll"; -#else - provider_options["backend_path"] = "libQnnHtp.so"; -#endif - provider_options["offload_graph_io_quantization"] = "0"; - - // Create QDQ models - std::vector onnx_model_paths{"./weight_share21.onnx", "./weight_share22.onnx"}; - // cleanup in case some failure test doesn't remove them - for (auto model_path : onnx_model_paths) { - std::remove(model_path.c_str()); - } - - std::vector ctx_model_paths; - for (auto model_path : onnx_model_paths) { - CreateQdqModel(model_path, DefaultLoggingManager().DefaultLogger()); - EXPECT_TRUE(std::filesystem::exists(model_path.c_str())); - auto pos = model_path.find_last_of("."); - if (pos != std::string::npos) { - model_path = model_path.substr(0, pos) + "_ctx.onnx"; - } else { - model_path = model_path + "_ctx.onnx"; - } - ctx_model_paths.push_back(model_path); - } - for (auto ctx_model_path : ctx_model_paths) { - std::remove(ctx_model_path.c_str()); - } - - DumpModelWithSharedCtx(provider_options, onnx_model_paths[0], onnx_model_paths[1]); - - std::string qnn_ctx_binary_file_name1; - GetContextBinaryFileName(ctx_model_paths[0], qnn_ctx_binary_file_name1, - DefaultLoggingManager().DefaultLogger()); - EXPECT_TRUE(!qnn_ctx_binary_file_name1.empty()); - - std::string qnn_ctx_binary_file_name2; - GetContextBinaryFileName(ctx_model_paths[1], qnn_ctx_binary_file_name2, - DefaultLoggingManager().DefaultLogger()); - EXPECT_TRUE(!qnn_ctx_binary_file_name2.empty()); - // 2 *_ctx.onn point to same .bin file - EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); - - Ort::SessionOptions so; - so.AddConfigEntry(kOrtSessionOptionShareEpContexts, "1"); - so.AppendExecutionProvider("QNN", provider_options); - - EXPECT_TRUE(2 == ctx_model_paths.size()); -#ifdef _WIN32 - std::wstring ctx_model_file1(ctx_model_paths[0].begin(), ctx_model_paths[0].end()); - std::wstring ctx_model_file2(ctx_model_paths[1].begin(), ctx_model_paths[1].end()); -#else - std::string ctx_model_file1(ctx_model_paths[0].begin(), ctx_model_paths[0].end()); - std::string ctx_model_file2(ctx_model_paths[1].begin(), ctx_model_paths[1].end()); -#endif - // Create session from the 2nd model first - Ort::Session session1(*ort_env, ctx_model_file2.c_str(), so); - Ort::Session session2(*ort_env, ctx_model_file1.c_str(), so); - - std::vector input_names; - std::vector output_names; - GetModelInputNames(ctx_model_paths[1], input_names, output_names, - DefaultLoggingManager().DefaultLogger()); - - // Run sessions - // prepare input - std::vector input_dim{2, 3}; - std::vector input_value(2 * 3, 0.0f); - Ort::MemoryInfo info("Cpu", OrtDeviceAllocator, 0, OrtMemTypeDefault); - std::vector ort_inputs; - std::vector input_names_c; - for (size_t i = 0; i < input_names.size(); ++i) { - auto input_tensor = Ort::Value::CreateTensor(info, input_value.data(), input_value.size(), - input_dim.data(), input_dim.size()); - ort_inputs.push_back(std::move(input_tensor)); - input_names_c.push_back(input_names[i].c_str()); - } - std::vector output_names_c; - for (size_t i = 0; i < output_names.size(); ++i) { - output_names_c.push_back(output_names[i].c_str()); - } - - auto ort_outputs1 = session1.Run(Ort::RunOptions{}, input_names_c.data(), ort_inputs.data(), ort_inputs.size(), - output_names_c.data(), 1); - - for (auto model_path : onnx_model_paths) { - std::remove(model_path.c_str()); - } - for (auto ctx_model_path : ctx_model_paths) { - std::remove(ctx_model_path.c_str()); - } - std::remove(qnn_ctx_binary_file_name1.c_str()); -} // For Ort sessions to generate the context binary, with session option ep.share_ep_contexts enabled // Ort sessions will share the QnnBackendManager, so that all graphs from all models compile into the same Qnn context @@ -1397,6 +1296,8 @@ TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { // 2 *_ctx.onn point to same .bin file EXPECT_TRUE(qnn_ctx_binary_file_name1 == qnn_ctx_binary_file_name2); + auto file_size_1 = std::filesystem::file_size(qnn_ctx_binary_file_name1); + EXPECT_TRUE(file_size_1 > 0); // clean up for (auto model_path : onnx_model_paths) { From 296787be0fef23e173145850b660f6e6492f2b79 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 11:39:27 -0700 Subject: [PATCH 11/13] format --- onnxruntime/test/providers/qnn/qnn_ep_context_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index dfe5b88632cf0..5d310cbb41531 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1236,7 +1236,7 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions) { auto ort_outputs1 = session1.Run(Ort::RunOptions{}, input_names_c.data(), ort_inputs.data(), ort_inputs.size(), output_names_c.data(), 1); - #endif +#endif for (auto model_path : onnx_model_paths) { std::remove(model_path.c_str()); @@ -1247,7 +1247,6 @@ TEST_F(QnnHTPBackendTests, QnnContextShareAcrossSessions) { std::remove(qnn_ctx_binary_file_name1.c_str()); } - // For Ort sessions to generate the context binary, with session option ep.share_ep_contexts enabled // Ort sessions will share the QnnBackendManager, so that all graphs from all models compile into the same Qnn context TEST_F(QnnHTPBackendTests, QnnContextGenWeightSharingSessionAPI) { From 08dee61cca9b5818d16e0aa449605ff690c07a93 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 14:18:47 -0700 Subject: [PATCH 12/13] resolve build issue on Linux --- onnxruntime/test/providers/qnn/qnn_ep_context_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc index 5d310cbb41531..e39102a21dd1c 100644 --- a/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_ep_context_test.cc @@ -1114,6 +1114,7 @@ static void DumpModelWithSharedCtx(ProviderOptions provider_options, Ort::Session session2(*ort_env, ToPathString(onnx_model_path2).c_str(), so); } +#if defined(__aarch64__) || defined(_M_ARM64) static void GetModelInputNames(const std::string& model_path, std::vector& input_names, std::vector& output_names, @@ -1133,6 +1134,7 @@ static void GetModelInputNames(const std::string& model_path, output_names.push_back(output->Name()); } } +#endif // 1. Create 2 QDQ models // 2. Initialize 2 Ort sessions which share the same QNN EP from these 2 QDQ models From 0fb6f62a435116872e08cacb02e08e738e80d838 Mon Sep 17 00:00:00 2001 From: Hector Li Date: Tue, 11 Mar 2025 18:31:05 -0700 Subject: [PATCH 13/13] remove comments not accurate --- onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc index 103559a141beb..92671e52f62f9 100644 --- a/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc +++ b/onnxruntime/test/ep_weight_sharing_ctx_gen/main.cc @@ -166,8 +166,6 @@ int real_main(int argc, char* argv[]) { // Only with enable_htp_spill_fill_buffer enabled: // Update generated context cache Onnx model to have the same max_size (align with the last generated model) // so that the inference session can be created with any order of the ctx.onnx models - // otherwise, user can also skip this step, but they need to create the inference session from the last generated ctx.onnx model - // since only the last ctx.onnx has EPContext nodes has the correct max_size covers graphs for all sessions const std::string enable_htp_spill_fill_buffer = "enable_htp_spill_fill_buffer"; auto pos = provider_options.find(enable_htp_spill_fill_buffer); if (pos != provider_options.end() && pos->second == "1") {