Skip to content

Add onnxruntime_plugin_ep_onnx_test for plugin EPs to run onnx test #25942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Chi Lo (chilo-ms) wants to merge 12 commits into
base: main
Choose a base branch
from
Open

Add onnxruntime_plugin_ep_onnx_test for plugin EPs to run onnx test #25942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c1e2871
initial commit
chilo-ms Sep 3, 2025
f27a79a
update
chilo-ms Sep 4, 2025
fd6f666
update
chilo-ms Sep 5, 2025
1963df4
update
chilo-ms Sep 5, 2025
e74412c
Fix issue of not saving Ort::Value but accessing it due to using GetC…
chilo-ms Sep 5, 2025
09ed403
fix build dependency issue
chilo-ms Sep 5, 2025
92b0368
fix warning
chilo-ms Sep 5, 2025
1b85262
fix warning
chilo-ms Sep 8, 2025
771f8da
add back macro check
chilo-ms Sep 8, 2025
30970c7
clean up the code
chilo-ms Sep 8, 2025
3492ba9
Merge branch 'main' into chi/onnx_test_runner_plugin_ep
chilo-ms Sep 9, 2025
7bbc027
fix build issue for DML EP Pipeline
chilo-ms Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions cmake/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1054,6 +1054,11 @@ function(onnxruntime_set_compile_flags target_name)
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:-Xcudafe \"--diag_suppress=conversion_function_not_usable\">")
target_compile_definitions(${target_name} PRIVATE -DDISABLE_CUSPARSE_DEPRECATED)
endif()

if (onnxruntime_BUILD_SHARED_LIB)
target_compile_definitions(${target_name} PRIVATE BUILD_SHARED_LIB)
endif()

if (MSVC)
foreach(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
target_compile_options(${target_name} PRIVATE "$<$<COMPILE_LANGUAGE:CXX,C>:/external:I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY}>")
Expand Down
84 changes: 82 additions & 2 deletions cmake/onnxruntime_unittests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,14 @@ file(GLOB onnxruntime_test_utils_src CONFIGURE_DEPENDS
"${TEST_SRC_DIR}/util/*.cc"
)

file(GLOB onnxruntime_test_utils_public_values_src CONFIGURE_DEPENDS
"${TEST_SRC_DIR}/util/values/public_api/*.cc"
)

file(GLOB onnxruntime_test_utils_internal_values_src CONFIGURE_DEPENDS
"${TEST_SRC_DIR}/util/values/internal_api/*.cc"
)

file(GLOB onnxruntime_test_common_src CONFIGURE_DEPENDS
"${TEST_SRC_DIR}/common/*.cc"
"${TEST_SRC_DIR}/common/*.h"
Expand Down Expand Up @@ -535,6 +543,7 @@ set (onnxruntime_webgpu_delay_load_test_SRC

set(onnxruntime_test_common_libs
onnxruntime_test_utils
onnxruntime_test_utils_internal_values
onnxruntime_common
)

Expand Down Expand Up @@ -790,6 +799,7 @@ else()
target_compile_options(onnxruntime_test_utils PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:-Wno-error=character-conversion>")
endif()
endif()

if (onnxruntime_USE_NCCL)
target_include_directories(onnxruntime_test_utils PRIVATE ${NCCL_INCLUDE_DIRS})
endif()
Expand All @@ -803,11 +813,27 @@ target_include_directories(onnxruntime_test_utils PUBLIC "${TEST_SRC_DIR}/util/i
set_target_properties(onnxruntime_test_utils PROPERTIES FOLDER "ONNXRuntimeTest")
source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_test_utils_src})

# This utility library uses onnxruntime c/c++ apis to provide capability of comparing two OrtValues with all supported data type.
onnxruntime_add_static_library(onnxruntime_test_utils_public_values ${onnxruntime_test_utils_public_values_src})
onnxruntime_add_include_to_target(onnxruntime_test_utils_public_values onnxruntime_common onnx onnx_proto Eigen3::Eigen)
target_include_directories(onnxruntime_test_utils_public_values PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT})
add_dependencies(onnxruntime_test_utils_public_values ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_test_utils_public_values PROPERTIES FOLDER "ONNXRuntimeTest")

# This utility library uses internal onnxruntime libraries, e.g. onnxruntime::Tensor, to provide capability of comparing two OrtValues with all supported data type.
onnxruntime_add_static_library(onnxruntime_test_utils_internal_values ${onnxruntime_test_utils_internal_values_src})
onnxruntime_add_include_to_target(onnxruntime_test_utils_internal_values onnxruntime_common onnx onnx_proto flatbuffers::flatbuffers Boost::mp11 Eigen3::Eigen)
target_include_directories(onnxruntime_test_utils_internal_values PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT} "${TEST_SRC_DIR}/util/include")
add_dependencies(onnxruntime_test_utils_internal_values ${onnxruntime_EXTERNAL_DEPENDENCIES})
set_target_properties(onnxruntime_test_utils_internal_values PROPERTIES FOLDER "ONNXRuntimeTest")

if(NOT IOS)
set(onnx_test_runner_src_dir ${TEST_SRC_DIR}/onnx)
file(GLOB onnx_test_runner_common_srcs CONFIGURE_DEPENDS
${onnx_test_runner_src_dir}/*.h
${onnx_test_runner_src_dir}/*.cc)
${onnx_test_runner_src_dir}/*.cc
${onnx_test_runner_src_dir}/utils/*.h
${onnx_test_runner_src_dir}/utils/*.cc)

list(REMOVE_ITEM onnx_test_runner_common_srcs ${onnx_test_runner_src_dir}/main.cc)

Expand Down Expand Up @@ -852,7 +878,7 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS)
onnxruntime_add_include_to_target(onnxruntime_providers_cuda_ut GTest::gtest GTest::gmock)
add_dependencies(onnxruntime_providers_cuda_ut onnxruntime_test_utils onnxruntime_common)
target_include_directories(onnxruntime_providers_cuda_ut PRIVATE ${ONNXRUNTIME_ROOT}/core/mickey)
target_link_libraries(onnxruntime_providers_cuda_ut PRIVATE GTest::gtest GTest::gmock ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_test_utils onnxruntime_common)
target_link_libraries(onnxruntime_providers_cuda_ut PRIVATE GTest::gtest GTest::gmock ${ONNXRUNTIME_MLAS_LIBS} onnxruntime_test_utils onnxruntime_test_utils_internal_values onnxruntime_common)
if (MSVC)
# Cutlass code has an issue with the following:
# warning C4100: 'magic': unreferenced formal parameter
Expand Down Expand Up @@ -1098,6 +1124,7 @@ endif()

set(onnx_test_libs
onnxruntime_test_utils
onnxruntime_test_utils_internal_values
${ONNXRUNTIME_TEST_LIBS}
onnx_test_data_proto
${onnxruntime_EXTERNAL_LIBRARIES})
Expand Down Expand Up @@ -1131,6 +1158,59 @@ if (NOT IOS)
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
endif()

if (NOT IOS)
onnxruntime_add_executable(onnxruntime_plugin_ep_onnx_test
${onnx_test_runner_src_dir}/plugin_ep/main.cc
${onnx_test_runner_src_dir}/plugin_ep/command_args_parser.cc
${onnx_test_runner_src_dir}/plugin_ep/command_args_parser.h)
if(MSVC)
target_compile_options(onnxruntime_plugin_ep_onnx_test PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>"
"$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")
endif()
if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
if (onnxruntime_ENABLE_WEBASSEMBLY_THREADS)
set_target_properties(onnxruntime_plugin_ep_onnx_test PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1 -s PROXY_TO_PTHREAD=1 -s EXIT_RUNTIME=1")
else()
set_target_properties(onnxruntime_plugin_ep_onnx_test PROPERTIES LINK_FLAGS "-s NODERAWFS=1 -s ALLOW_MEMORY_GROWTH=1")
endif()
endif()

# ABSL_FLAGS_STRIP_NAMES is set to 1 by default to disable flag registration when building for Android, iPhone, and "embedded devices".
# See the issue: https://github.com/abseil/abseil-cpp/issues/1875
# We set it to 0 for all builds to be able to use ABSL flags for onnxruntime_plugin_ep_onnx_test.
target_compile_definitions(onnxruntime_plugin_ep_onnx_test PRIVATE ABSL_FLAGS_STRIP_NAMES=0)

if (onnxruntime_BUILD_SHARED_LIB)
set(onnx_test_runner_libs
onnx_test_runner_common onnxruntime_test_utils onnxruntime_test_utils_public_values onnxruntime_common
onnxruntime onnxruntime_flatbuffers onnx_test_data_proto
${onnxruntime_EXTERNAL_LIBRARIES}
absl::flags absl::flags_parse ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})

target_link_libraries(onnxruntime_plugin_ep_onnx_test PRIVATE ${onnx_test_runner_libs} nlohmann_json::nlohmann_json)

if(WIN32)
target_link_libraries(onnxruntime_plugin_ep_onnx_test PRIVATE debug dbghelp advapi32)
endif()
else()
target_link_libraries(onnxruntime_plugin_ep_onnx_test PRIVATE onnx_test_runner_common absl::flags absl::flags_parse ${onnx_test_libs} nlohmann_json::nlohmann_json)
endif()

target_include_directories(onnxruntime_plugin_ep_onnx_test PRIVATE ${ONNXRUNTIME_ROOT})
target_include_directories(onnxruntime_plugin_ep_onnx_test PRIVATE ${onnx_test_runner_src_dir})

if (onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
target_link_libraries(onnxruntime_plugin_ep_onnx_test PRIVATE Python::Python)
endif()
set_target_properties(onnxruntime_plugin_ep_onnx_test PROPERTIES FOLDER "ONNXRuntimeTest")

install(TARGETS onnxruntime_plugin_ep_onnx_test
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
BUNDLE DESTINATION ${CMAKE_INSTALL_LIBDIR}
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
endif()

if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if(onnxruntime_BUILD_BENCHMARKS)
SET(BENCHMARK_DIR ${TEST_SRC_DIR}/onnx/microbenchmark)
Expand Down
13 changes: 7 additions & 6 deletions onnxruntime/test/onnx/dataitem_request.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "heap_buffer.h"
#include "TestCase.h"
#include "test/compare_ortvalue.h"
#include "utils/macros.h"

#include "core/common/logging/logging.h"
#include "core/common/common.h"
Expand All @@ -29,7 +30,7 @@ std::pair<EXECUTE_RESULT, TIME_SPEC> DataTaskRequestContext::Run(const ITestCase
ORT_CATCH(const std::exception& ex) {
ORT_HANDLE_EXCEPTION([&]() {
result = std::make_pair(EXECUTE_RESULT::WITH_EXCEPTION, ctx.GetTimeSpent());
LOGS_DEFAULT(ERROR) << ctx.test_case_.GetTestCaseName() << ":" << ex.what();
TEST_LOG_ERROR(ctx.test_case_.GetTestCaseName() + ":" + ex.what());
});
}
return result;
Expand All @@ -54,7 +55,7 @@ void DataTaskRequestContext::RunAsync() {
ORT_CATCH(const std::exception& ex) {
ORT_HANDLE_EXCEPTION([&]() {
result = std::make_pair(EXECUTE_RESULT::WITH_EXCEPTION, spent_time_);
LOGS_DEFAULT(ERROR) << test_case_.GetTestCaseName() << ":" << ex.what();
TEST_LOG_ERROR(test_case_.GetTestCaseName() + ":" + ex.what());
});
}

Expand Down Expand Up @@ -145,7 +146,7 @@ std::pair<EXECUTE_RESULT, TIME_SPEC> DataTaskRequestContext::RunImpl() {
auto iter = name_fetch_output_map.find(output_name);
if (iter == name_fetch_output_map.end()) {
res = EXECUTE_RESULT::INVALID_GRAPH;
LOGF_DEFAULT(ERROR, "cannot find %s in the outputs", output_name.c_str());
TEST_LOG_ERROR("cannot find %s in the outputs" + output_name);
break;
}
OrtValue* actual_output_value = iter->second;
Expand All @@ -155,7 +156,7 @@ std::pair<EXECUTE_RESULT, TIME_SPEC> DataTaskRequestContext::RunImpl() {
// Expected output is not None
if (expected_output_value != nullptr) {
// Actual output is None
if (!actual_output_value->IsAllocated()) {
if (!Ort::ConstValue(actual_output_value).HasValue()) {
ret = std::pair<COMPARE_RESULT, std::string>{
COMPARE_RESULT::RESULT_DIFFERS,
"Expected non-None output but received an OrtValue that is None"};
Expand All @@ -165,7 +166,7 @@ std::pair<EXECUTE_RESULT, TIME_SPEC> DataTaskRequestContext::RunImpl() {
relative_per_sample_tolerance, post_procesing);
}
} else { // Expected output is None, ensure that the received output OrtValue is None as well
if (actual_output_value->IsAllocated()) {
if (Ort::ConstValue(actual_output_value).HasValue()) {
ret = std::pair<COMPARE_RESULT, std::string>{
COMPARE_RESULT::RESULT_DIFFERS,
"Expected None output but received an OrtValue that is not None"};
Expand Down Expand Up @@ -213,7 +214,7 @@ std::pair<EXECUTE_RESULT, TIME_SPEC> DataTaskRequestContext::RunImpl() {
}

if (compare_result != COMPARE_RESULT::SUCCESS && !ret.second.empty()) {
LOGS_DEFAULT(ERROR) << test_case_.GetTestCaseName() << ":output=" << output_name << ":" << ret.second;
TEST_LOG_ERROR(test_case_.GetTestCaseName() + ":output=" + output_name + ":" + ret.second);
}
if (compare_result != COMPARE_RESULT::SUCCESS) {
break;
Expand Down
Loading
Loading