Skip to content

QNN-EP: DSPQueue Polling #25361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Hector Li (HectorSVC) merged 2 commits into from
Jul 11, 2025
Merged

QNN-EP: DSPQueue Polling #25361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ace53c
[QNN-EP] Enable DSPQueue Polling for Burst
quic-calvnguy Jun 27, 2025
118cd8f
small optimization with QNN context config
quic-calvnguy Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions onnxruntime/core/providers/qnn/builder/qnn_backend_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1426,13 +1426,33 @@ Status QnnBackendManager::SetHtpPowerConfig(uint32_t htp_power_config_client_id,
return Status::OK();
}

Status QnnBackendManager::SetRpcControlLatency(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency) {
Status QnnBackendManager::SetRpcPowerConfigs(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency,
uint32_t rpc_polling_time) {
// This function is called in QNN EP's OnRunStart() even if QNN backend setup failed and the model is assigned
// to a different EP. Therefore, we have to check that backend setup actually completed before trying to
// set RPC control latency. Otherwise, this causes a segfault because the QNN backend library is unloaded.
ORT_RETURN_IF_NOT(backend_setup_completed_, "Cannot set HTP RPC control latency if backend setup is not complete.");

constexpr int kNumRpcPollingPowerConfigs = 2;
std::vector<QnnHtpPerfInfrastructure_PowerConfig_t> rpc_power_configs;
rpc_power_configs.reserve(kNumRpcPollingPowerConfigs);

// Set rpc control latency here
if (rpc_control_latency != 0) {
auto& rpc_control_latency_cfg = rpc_power_configs.emplace_back();
rpc_control_latency_cfg.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY;
rpc_control_latency_cfg.rpcControlLatencyConfig = rpc_control_latency;
}

// Note: v68 does not support rpc polling mode
if (rpc_polling_time != 0) {
auto& rpc_polling_time_cfg = rpc_power_configs.emplace_back();
rpc_polling_time_cfg.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME;
rpc_polling_time_cfg.rpcPollingTimeConfig = rpc_polling_time;
}

if (rpc_power_configs.size() > 0) {
QnnDevice_Infrastructure_t qnn_device_infra = nullptr;
auto status = qnn_interface_.deviceGetInfrastructure(&qnn_device_infra);
ORT_RETURN_IF(QNN_SUCCESS != status, "backendGetPerfInfrastructure failed.");
Expand All @@ -1442,15 +1462,6 @@ Status QnnBackendManager::SetRpcControlLatency(uint32_t htp_power_config_client_
"HTP infra type = ", htp_infra->infraType, ", which is not perf infra type.");
QnnHtpDevice_PerfInfrastructure_t& htp_perf_infra = htp_infra->perfInfra;

// Set rpc control latency here, but note that v68 doesn't support rpc polling mode.
constexpr int kNumRpcPollingPowerConfigs = 2;
std::vector<QnnHtpPerfInfrastructure_PowerConfig_t> rpc_power_configs(kNumRpcPollingPowerConfigs);
QnnHtpPerfInfrastructure_PowerConfig_t& rpc_control_latency_cfg = rpc_power_configs[0];
// v68 doesn't support this.
QnnHtpPerfInfrastructure_PowerConfig_t& rpc_polling_time = rpc_power_configs[1];
rpc_control_latency_cfg.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_CONTROL_LATENCY;
rpc_polling_time.option = QNN_HTP_PERF_INFRASTRUCTURE_POWER_CONFIGOPTION_RPC_POLLING_TIME;
rpc_control_latency_cfg.rpcControlLatencyConfig = rpc_control_latency;
std::vector<const QnnHtpPerfInfrastructure_PowerConfig_t*> perf_power_configs_ptr =
ObtainNullTermPtrVector(rpc_power_configs);
status = htp_perf_infra.setPowerConfig(htp_power_config_client_id, perf_power_configs_ptr.data());
Expand Down
5 changes: 3 additions & 2 deletions onnxruntime/core/providers/qnn/builder/qnn_backend_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,9 @@ class QnnBackendManager : public std::enable_shared_from_this<QnnBackendManager>
Status SetHtpPowerConfig(uint32_t htp_power_config_client_id,
HtpPerformanceMode htp_performance_mode);

Status SetRpcControlLatency(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency);
Status SetRpcPowerConfigs(uint32_t htp_power_config_client_id,
uint32_t rpc_control_latency,
uint32_t rpc_polling_time);

const QNN_INTERFACE_VER_TYPE& GetQnnInterface() { return qnn_interface_; }

Expand Down
25 changes: 17 additions & 8 deletions onnxruntime/core/providers/qnn/qnn_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1356,7 +1356,8 @@ QNNExecutionProvider::PerThreadContext::PerThreadContext(qnn::QnnBackendManager*
uint32_t device_id,
uint32_t core_id,
qnn::HtpPerformanceMode default_htp_performance_mode,
uint32_t default_rpc_control_latency)
uint32_t default_rpc_control_latency,
uint32_t default_rpc_polling_time)
: qnn_backend_manager_(qnn_backend_manager) {
Status rt = qnn_backend_manager_->CreateHtpPowerCfgId(device_id, core_id, htp_power_config_id_);
is_htp_power_config_id_valid_ = rt.IsOK();
Expand All @@ -1367,9 +1368,10 @@ QNNExecutionProvider::PerThreadContext::PerThreadContext(qnn::QnnBackendManager*
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->SetHtpPowerConfig(htp_power_config_id_,
default_htp_performance_mode));
}
if (default_rpc_control_latency > 0) {
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->SetRpcControlLatency(htp_power_config_id_,
default_rpc_control_latency));
if (default_rpc_control_latency > 0 || default_rpc_polling_time > 0) {
ORT_IGNORE_RETURN_VALUE(qnn_backend_manager_->SetRpcPowerConfigs(htp_power_config_id_,
default_rpc_control_latency,
default_rpc_polling_time));
}
}
}
Expand Down Expand Up @@ -1400,7 +1402,8 @@ QNNExecutionProvider::PerThreadContext& QNNExecutionProvider::GetPerThreadContex
if (context_state_.retired_context_pool.empty()) {
uint32_t core_id = 0;
context = std::make_shared<PerThreadContext>(qnn_backend_manager_.get(), device_id_, core_id,
default_htp_performance_mode_, default_rpc_control_latency_);
default_htp_performance_mode_, default_rpc_control_latency_,
default_rpc_polling_time_);
} else {
context = context_state_.retired_context_pool.back();
context_state_.retired_context_pool.pop_back();
Expand Down Expand Up @@ -1468,15 +1471,21 @@ Status QNNExecutionProvider::OnRunStart(const onnxruntime::RunOptions& run_optio
LOGS_DEFAULT(VERBOSE) << "rpc_control_latency: " << rpc_control_latency;
}

uint32_t rpc_polling_time = 0;
if (qnn::HtpPerformanceMode::kHtpBurst != htp_performance_mode) {
Comment thread
HectorSVC marked this conversation as resolved.
rpc_polling_time = 9999;
Comment thread
HectorSVC marked this conversation as resolved.
}

if (GetPerThreadContext().IsHtpPowerConfigIdValid()) {
if (qnn::HtpPerformanceMode::kHtpDefault != htp_performance_mode) {
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetHtpPowerConfig(GetPerThreadContext().GetHtpPowerConfigId(),
htp_performance_mode));
}

if (rpc_control_latency > 0) {
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetRpcControlLatency(GetPerThreadContext().GetHtpPowerConfigId(),
rpc_control_latency));
if (rpc_control_latency > 0 || rpc_polling_time > 0) {
ORT_RETURN_IF_ERROR(qnn_backend_manager_->SetRpcPowerConfigs(GetPerThreadContext().GetHtpPowerConfigId(),
rpc_control_latency,
rpc_polling_time));
}
}

Expand Down
4 changes: 3 additions & 1 deletion onnxruntime/core/providers/qnn/qnn_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ class QNNExecutionProvider : public IExecutionProvider {
uint32_t device_id_ = 0;
qnn::HtpPerformanceMode default_htp_performance_mode_ = qnn::HtpPerformanceMode::kHtpDefault;
uint32_t default_rpc_control_latency_ = 0;
uint32_t default_rpc_polling_time_ = 0;
bool enable_HTP_FP16_precision_ = true;
bool share_ep_contexts_ = false;
bool stop_share_ep_contexts_ = false;
Expand All @@ -116,7 +117,8 @@ class QNNExecutionProvider : public IExecutionProvider {
PerThreadContext(qnn::QnnBackendManager* qnn_backend_manager,
uint32_t device_id, uint32_t core_id,
qnn::HtpPerformanceMode default_htp_performance_mode,
uint32_t default_rpc_control_latency);
uint32_t default_rpc_control_latency,
uint32_t default_rpc_polling_time);
~PerThreadContext();
ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(PerThreadContext);

Expand Down
Loading