diff --git a/.github/labeler.yml b/.github/labeler.yml index c14e2a213bc60..21ca6769d491c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -11,7 +11,6 @@ ep:oneDNN: '/\bone\s*dnn\b/i' ep:OpenVINO: '/\bopen\s*vino\b/i' ep:QNN: '/\bqnn\b/i' ep:RockchipNPU: '/\brockchip(?:npu)?\b/i' -ep:ROCm: '/\brocm\b/i' ep:SNPE: '/\bsnpe\b/i' ep:tvm: '/\btvm\b/i' ep:VitisAI: '/\bvitis(?:ai)?\b/i' diff --git a/ThirdPartyNotices.txt b/ThirdPartyNotices.txt index 7b2bbdd2094d1..fbd9f9a95f601 100644 --- a/ThirdPartyNotices.txt +++ b/ThirdPartyNotices.txt @@ -5806,41 +5806,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. _____ -composable_kernel - -https://github.com/ROCmSoftwarePlatform/composable_kernel - -Copyright (c) 2018- , Advanced Micro Devices, Inc. (Chao Liu, Jing Zhang) -Copyright (c) 2019- , Advanced Micro Devices, Inc. (Letao Qin, Qianfeng Zhang, Liang Huang, Shaojie Wang) -Copyright (c) 2022- , Advanced Micro Devices, Inc. (Anthony Chang, Chunyu Lai, Illia Silin, Adam Osewski, Poyen Chen, Jehandad Khan) -Copyright (c) 2019-2021, Advanced Micro Devices, Inc. (Hanwen Chang) -Copyright (c) 2019-2020, Advanced Micro Devices, Inc. (Tejash Shah) -Copyright (c) 2020 , Advanced Micro Devices, Inc. (Xiaoyan Zhou) -Copyright (c) 2021-2022, Advanced Micro Devices, Inc. (Jianfeng Yan) - -SPDX-License-Identifier: MIT -Copyright (c) 2018-2023, Advanced Micro Devices, Inc. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -_____ - neural-speed https://github.com/intel/neural-speed diff --git a/cmake/external/composable_kernel.cmake b/cmake/external/composable_kernel.cmake deleted file mode 100644 index 826bb7c468a02..0000000000000 --- a/cmake/external/composable_kernel.cmake +++ /dev/null @@ -1,66 +0,0 @@ -set(PATCH_CLANG ${PROJECT_SOURCE_DIR}/patches/composable_kernel/Fix_Clang_Build.patch) -set(PATCH_GFX12X ${PROJECT_SOURCE_DIR}/patches/composable_kernel/Add_gfx12x_support.patch) - -include(FetchContent) -onnxruntime_fetchcontent_declare(composable_kernel - URL ${DEP_URL_composable_kernel} - URL_HASH SHA1=${DEP_SHA1_composable_kernel} - PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PATCH_CLANG} && - ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PATCH_GFX12X} - EXCLUDE_FROM_ALL -) - -FetchContent_GetProperties(composable_kernel) -if(NOT composable_kernel_POPULATED) - FetchContent_Populate(composable_kernel) - set(GPU_TARGETS ${CMAKE_HIP_ARCHITECTURES}) - set(BUILD_DEV OFF CACHE BOOL "Disable -Weverything, otherwise, error: 'constexpr' specifier is incompatible with C++98 [-Werror,-Wc++98-compat]" FORCE) - # Exclude i8 device gemm instances due to excessive long compilation time and not being used - set(DTYPES fp32 fp16 bf16 fp8) - set(INSTANCES_ONLY ON) - add_subdirectory(${composable_kernel_SOURCE_DIR} ${composable_kernel_BINARY_DIR} EXCLUDE_FROM_ALL) - - add_library(onnxruntime_composable_kernel_includes INTERFACE) - target_include_directories(onnxruntime_composable_kernel_includes INTERFACE - ${composable_kernel_SOURCE_DIR}/include - ${composable_kernel_BINARY_DIR}/include - ${composable_kernel_SOURCE_DIR}/library/include) - target_compile_definitions(onnxruntime_composable_kernel_includes INTERFACE __fp32__ __fp16__ __bf16__) - - execute_process( - COMMAND ${Python3_EXECUTABLE} ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/generate.py - --list_blobs ${composable_kernel_BINARY_DIR}/blob_list.txt - COMMAND_ERROR_IS_FATAL ANY - ) - file(STRINGS ${composable_kernel_BINARY_DIR}/blob_list.txt generated_fmha_srcs) - add_custom_command( - OUTPUT ${generated_fmha_srcs} - COMMAND ${Python3_EXECUTABLE} ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/generate.py --output_dir ${composable_kernel_BINARY_DIR} - DEPENDS ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/generate.py ${composable_kernel_BINARY_DIR}/blob_list.txt - ) - set_source_files_properties(${generated_fmha_srcs} PROPERTIES LANGUAGE HIP GENERATED TRUE) - add_custom_target(gen_fmha_srcs DEPENDS ${generated_fmha_srcs}) # dummy target for dependencies - # code generation complete - - set(fmha_srcs - ${generated_fmha_srcs} - ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/fmha_fwd.cpp - ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/fmha_fwd.hpp - ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/bias.hpp - ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha/mask.hpp - ) - add_library(onnxruntime_composable_kernel_fmha STATIC EXCLUDE_FROM_ALL ${generated_fmha_srcs}) - target_link_libraries(onnxruntime_composable_kernel_fmha PUBLIC onnxruntime_composable_kernel_includes) - target_include_directories(onnxruntime_composable_kernel_fmha PUBLIC ${composable_kernel_SOURCE_DIR}/example/ck_tile/01_fmha) - add_dependencies(onnxruntime_composable_kernel_fmha gen_fmha_srcs) - - # ck tile only supports MI200+ GPUs at the moment - get_target_property(archs onnxruntime_composable_kernel_fmha HIP_ARCHITECTURES) - string(REPLACE "," ";" archs "${archs}") - set(original_archs ${archs}) - list(FILTER archs INCLUDE REGEX "(gfx942|gfx90a)") - if (NOT original_archs EQUAL archs) - message(WARNING "ck tile only supports archs: ${archs} among the originally specified ${original_archs}") - endif() - set_target_properties(onnxruntime_composable_kernel_fmha PROPERTIES HIP_ARCHITECTURES "${archs}") -endif() diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index e1d98109208d4..1dcc7553fd608 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -42,7 +42,7 @@ function(get_c_cxx_api_headers HEADERS_VAR) foreach(f ${ONNXRUNTIME_PROVIDER_NAMES}) # The header files in include/onnxruntime/core/providers/cuda directory cannot be flattened to the same directory # with onnxruntime_c_api.h . Most other EPs probably also do not work in this way. - if((NOT f STREQUAL cuda) AND (NOT f STREQUAL rocm)) + if(NOT f STREQUAL cuda) file(GLOB _provider_headers CONFIGURE_DEPENDS "${REPO_ROOT}/include/onnxruntime/core/providers/${f}/*.h" ) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index e449bb107e77b..1456c8caa8993 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -104,10 +104,6 @@ endif() if(onnxruntime_USE_CANN) target_include_directories(onnxruntime_pybind11_state PRIVATE ${onnxruntime_CANN_HOME}/include) endif() -if(onnxruntime_USE_ROCM) - target_compile_options(onnxruntime_pybind11_state PUBLIC -D__HIP_PLATFORM_AMD__=1 -D__HIP_PLATFORM_HCC__=1) - target_include_directories(onnxruntime_pybind11_state PRIVATE ${onnxruntime_ROCM_HOME}/hipfft/include ${onnxruntime_ROCM_HOME}/include ${onnxruntime_ROCM_HOME}/hiprand/include ${onnxruntime_ROCM_HOME}/rocrand/include ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/onnxruntime ${CMAKE_CURRENT_BINARY_DIR}/amdgpu/orttraining) -endif() if (onnxruntime_USE_NCCL) target_include_directories(onnxruntime_pybind11_state PRIVATE ${NCCL_INCLUDE_DIRS}) endif() @@ -774,7 +770,6 @@ endif() if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS|visionOS|tvOS" AND NOT CMAKE_SYSTEM_NAME STREQUAL "Android" - AND NOT onnxruntime_USE_ROCM AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") add_custom_command( TARGET onnxruntime_pybind11_state POST_BUILD @@ -1044,16 +1039,6 @@ if (onnxruntime_USE_CANN) ) endif() -if (onnxruntime_USE_ROCM) - add_custom_command( - TARGET onnxruntime_pybind11_state POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - $ - $ - $/onnxruntime/capi/ - ) -endif() - if (onnxruntime_USE_DML) if (NOT onnxruntime_USE_CUSTOM_DIRECTML) set(dml_shared_lib_path ${DML_PACKAGE_DIR}/bin/${onnxruntime_target_platform}-win/${DML_SHARED_LIB}) diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index f81a7a9726b76..86e5810952a09 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -73,6 +73,3 @@ if (NOT onnxruntime_BUILD_SHARED_LIB) FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() -if (onnxruntime_USE_NCCL AND onnxruntime_USE_ROCM) - add_dependencies(onnxruntime_session generate_hipified_files) -endif() diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 4913d38939792..397c37f5b5a94 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -677,7 +677,7 @@ if(onnxruntime_USE_ARMNN) endif() set(ONNXRUNTIME_TEST_STATIC_PROVIDER_LIBS - # CUDA, ROCM, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime. + # CUDA, TENSORRT, MIGRAPHX, DNNL, and OpenVINO are dynamically loaded at runtime. # QNN EP can be built as either a dynamic and static libs. ${PROVIDERS_NNAPI} ${PROVIDERS_VSINPU} diff --git a/cmake/patches/composable_kernel/Add_gfx12x_support.patch b/cmake/patches/composable_kernel/Add_gfx12x_support.patch deleted file mode 100644 index ef529184d2ed8..0000000000000 --- a/cmake/patches/composable_kernel/Add_gfx12x_support.patch +++ /dev/null @@ -1,2280 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index bc326c8b5..db5ad5052 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -117,7 +117,7 @@ else() - add_definitions(-DPROFILER_ONLY) - set(GPU_TARGETS "" CACHE STRING "" FORCE) - if(GPU_TARGETS) -- message(FATAL_ERROR "For PROFILE_ONLY build, please do not set GPU_TARGETS, use GPU_ARCH = gfx90, gfx94, gfx10, or gfx11") -+ message(FATAL_ERROR "For PROFILE_ONLY build, please do not set GPU_TARGETS, use GPU_ARCH = gfx90, gfx94, gfx10, gfx11 or gfx12") - endif() - if(GPU_ARCH MATCHES "gfx90") - rocm_check_target_ids(DEFAULT_GPU_TARGETS TARGETS "gfx908;gfx90a") -@@ -127,8 +127,10 @@ else() - rocm_check_target_ids(DEFAULT_GPU_TARGETS TARGETS "gfx1030") - elseif(GPU_ARCH MATCHES "gfx11") - rocm_check_target_ids(DEFAULT_GPU_TARGETS TARGETS "gfx1100;gfx1101;gfx1102") -+ elseif(GPU_ARCH MATCHES "gfx12") -+ rocm_check_target_ids(DEFAULT_GPU_TARGETS TARGETS "gfx1200;gfx1201") - else() -- message(FATAL_ERROR "For PROFILE_ONLY build, please specify GPU_ARCH as gfx90, gfx94, gfx10, or gfx11") -+ message(FATAL_ERROR "For PROFILE_ONLY build, please specify GPU_ARCH as gfx90, gfx94, gfx10, gfx11 or gfx12") - endif() - set(GPU_TARGETS "${DEFAULT_GPU_TARGETS}" CACHE STRING " " FORCE) - endif() -diff --git a/Jenkinsfile b/Jenkinsfile -index 75800bfc9..b72e2ca4e 100644 ---- a/Jenkinsfile -+++ b/Jenkinsfile -@@ -493,6 +493,7 @@ def Build_CK(Map conf=[:]){ - - def variant = env.STAGE_NAME - def retimage -+ - gitStatusWrapper(credentialsId: "${env.status_wrapper_creds}", gitHubContext: "Jenkins - ${variant}", account: 'ROCm', repo: 'composable_kernel') { - try { - (retimage, image) = getDockerImage(conf) -@@ -660,9 +661,6 @@ CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;ROCM - - pipeline { - agent none -- triggers { -- parameterizedCron(CRON_SETTINGS) -- } - options { - parallelsAlwaysFailFast() - } -diff --git a/cmake/EnableCompilerWarnings.cmake b/cmake/EnableCompilerWarnings.cmake -index 8654170b3..42070051b 100644 ---- a/cmake/EnableCompilerWarnings.cmake -+++ b/cmake/EnableCompilerWarnings.cmake -@@ -66,7 +66,7 @@ else() - -Wunreachable-code - -Wunused - -Wno-reserved-identifier -- -Werror -+ -Werror - -Wno-option-ignored - -Wsign-compare - -Wno-extra-semi-stmt -diff --git a/example/01_gemm/gemm_wmma_fp16.cpp b/example/01_gemm/gemm_wmma_fp16.cpp -index 8c52e4f7d..f8afe8d6d 100644 ---- a/example/01_gemm/gemm_wmma_fp16.cpp -+++ b/example/01_gemm/gemm_wmma_fp16.cpp -@@ -23,45 +23,45 @@ static constexpr auto GemmDefault = ck::tensor_operation::device::GemmSpecializa - - // clang-format off - using DeviceGemmInstance = ck::tensor_operation::device::DeviceGemmWmma_CShuffle -- < ALayout, -- BLayout, -- CLayout, -- ADataType, -+ < ALayout, -+ BLayout, -+ CLayout, -+ ADataType, - BDataType, -- CDataType, -- AccDataType, -- CShuffleDataType, -- AElementOp, -- BElementOp, -- CElementOp, -- GemmDefault, -+ CDataType, -+ AccDataType, -+ CShuffleDataType, -+ AElementOp, -+ BElementOp, -+ CElementOp, -+ GemmDefault, - 1, // Prefetch stage - 128, // BlockSize - 64, // MPerBlock - 128, // NPerBlock - 64, // KPerBlock -- 8, // K1 -+ 2, // K1 - 16, // MPerWmma - 16, // NPerWmma - 2, // M-Repeat // M-PerWmma / M-Repeat = M-Wave - 4, // N-Repeat // N-PerWmma / N-Repeat = N-Wave -- S<4, 32, 1>, -- S<1, 0, 2>, -- S<1, 0, 2>, -- 2, -- 8, -- 8, -- true, -- S<4, 32, 1>, -- S<1, 0, 2>, -- S<1, 0, 2>, -- 2, -- 8, -- 8, -- true, -+ S<4, 32, 1>, -+ S<1, 0, 2>, -+ S<1, 0, 2>, -+ 2, -+ 2, -+ 2, -+ true, -+ S<4, 32, 1>, -+ S<1, 0, 2>, -+ S<1, 0, 2>, -+ 2, -+ 2, -+ 2, -+ true, - 1, // C shuffle (M Repeat) Per store - 1, // C shuffle (N Repeat) Per store -- S<1, 32, 1, 4>, -+ S<1, 32, 1, 4>, - 8>; - // clang-format on - -diff --git a/example/01_gemm/run_gemm_example.inc b/example/01_gemm/run_gemm_example.inc -index b04e4e53a..cb15186c3 100644 ---- a/example/01_gemm/run_gemm_example.inc -+++ b/example/01_gemm/run_gemm_example.inc -@@ -159,7 +159,7 @@ bool run_gemm(const ProblemType& problem_size, const ExecutionConfig& config) - ck::utils::FillUniformDistributionIntegerValue{-5.f, 5.f}(b_k_n); - break; - case 4: -- ck::utils::FillUniformDistributionIntegerValue{1.f, 1.f}(a_m_k); -+ ck::utils::FillUniformDistributionIntegerValue{-5.f, 5.f}(a_m_k); - ck::utils::FillUniformDistributionIntegerValue{1.f, 1.f}(b_k_n); - break; - case 5: -diff --git a/example/04_gemm_add_add_fastgelu/CMakeLists.txt b/example/04_gemm_add_add_fastgelu/CMakeLists.txt -index ab19f819e..be47665a2 100644 ---- a/example/04_gemm_add_add_fastgelu/CMakeLists.txt -+++ b/example/04_gemm_add_add_fastgelu/CMakeLists.txt -@@ -24,4 +24,4 @@ foreach(gpu IN LISTS GPU_TARGETS) - add_example_dependencies(example_gemm_add_add_fastgelu_xdl example_gemm_add_add_fastgelu_xdl_lds_direct_load_fp32) - set(target 1) - endif() --endforeach() -\ No newline at end of file -+endforeach() -diff --git a/example/29_batched_gemm_bias_e_permute/batched_gemm_bias_e_permute_wmma_fp16.cpp b/example/29_batched_gemm_bias_e_permute/batched_gemm_bias_e_permute_wmma_fp16.cpp -index 2bbf430c4..f556be887 100644 ---- a/example/29_batched_gemm_bias_e_permute/batched_gemm_bias_e_permute_wmma_fp16.cpp -+++ b/example/29_batched_gemm_bias_e_permute/batched_gemm_bias_e_permute_wmma_fp16.cpp -@@ -83,14 +83,14 @@ using DeviceOpInstanceKKNN = - 2, - 4, - 4, -- true, -+ false, - S<4, 32, 1>, - S<1, 0, 2>, - S<1, 0, 2>, - 2, - 4, - 4, -- true, -+ false, - 1, - 1, - S<1, 64, 1, 2>, -diff --git a/example/32_batched_gemm_scale_softmax_gemm/cross_attention_forward_wmma_fp16.cpp b/example/32_batched_gemm_scale_softmax_gemm/cross_attention_forward_wmma_fp16.cpp -index 4c92c5497..fac19f8b5 100644 ---- a/example/32_batched_gemm_scale_softmax_gemm/cross_attention_forward_wmma_fp16.cpp -+++ b/example/32_batched_gemm_scale_softmax_gemm/cross_attention_forward_wmma_fp16.cpp -@@ -71,7 +71,7 @@ static constexpr auto TensorSpecC = ck::tensor_operation::device::TensorSpecial - #define CK_MHA_USE_WAVE_1 - #define CK_MHA_USE_WAVE_2 - #define CK_MHA_USE_WAVE_4 --#define CK_MHA_USE_WAVE_8 -+//#define CK_MHA_USE_WAVE_8 - using DeviceMHAFactory = - std::tuple< - #ifdef CK_MHA_USE_WAVE_1 -@@ -277,10 +277,10 @@ using DeviceMHAFactory = - S<2, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, false, - // CShuffleBlockTransfer MN - 1, 1, S<1, 64, 1, 2>, 8, -- MaskingSpec>, -+ MaskingSpec> - #endif - #ifdef CK_MHA_USE_WAVE_8 -- ck::tensor_operation::device::DeviceBatchedGemmSoftmaxGemmPermute_Wmma_CShuffle< -+ ,ck::tensor_operation::device::DeviceBatchedGemmSoftmaxGemmPermute_Wmma_CShuffle< - NumDimG, NumDimM, NumDimN, NumDimK, NumDimO, - ADataType, B0DataType, B1DataType, CDataType, Acc0BiasDataType, Acc0DataType, Acc1BiasDataType, Acc1DataType, CShuffleDataType, - AElementOp, B0ElementOp, Acc0ElementOp, B1ElementOp, CElementOp, -diff --git a/example/32_batched_gemm_scale_softmax_gemm/self_attention_forward_wmma_fp16.cpp b/example/32_batched_gemm_scale_softmax_gemm/self_attention_forward_wmma_fp16.cpp -index 8e037272b..d463cc871 100644 ---- a/example/32_batched_gemm_scale_softmax_gemm/self_attention_forward_wmma_fp16.cpp -+++ b/example/32_batched_gemm_scale_softmax_gemm/self_attention_forward_wmma_fp16.cpp -@@ -71,7 +71,7 @@ static constexpr auto TensorSpecC = ck::tensor_operation::device::TensorSpecial - #define CK_MHA_USE_WAVE_1 - #define CK_MHA_USE_WAVE_2 - #define CK_MHA_USE_WAVE_4 --#define CK_MHA_USE_WAVE_8 -+//#define CK_MHA_USE_WAVE_8 - using DeviceMHAFactory = - std::tuple< - #ifdef CK_MHA_USE_WAVE_1 -@@ -277,10 +277,10 @@ using DeviceMHAFactory = - S<2, 8, 8>, S<0, 2, 1>, S<0, 2, 1>, 1, 2, 1, false, - // CShuffleBlockTransfer MN - 1, 1, S<1, 64, 1, 2>, 8, -- MaskingSpec>, -+ MaskingSpec> - #endif - #ifdef CK_MHA_USE_WAVE_8 -- ck::tensor_operation::device::DeviceBatchedGemmSoftmaxGemmPermute_Wmma_CShuffle< -+ ,ck::tensor_operation::device::DeviceBatchedGemmSoftmaxGemmPermute_Wmma_CShuffle< - NumDimG, NumDimM, NumDimN, NumDimK, NumDimO, - ADataType, B0DataType, B1DataType, CDataType, Acc0BiasDataType, Acc0DataType, Acc1BiasDataType, Acc1DataType, CShuffleDataType, - AElementOp, B0ElementOp, Acc0ElementOp, B1ElementOp, CElementOp, -diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt -index 5465adb77..7534bff3b 100644 ---- a/example/CMakeLists.txt -+++ b/example/CMakeLists.txt -@@ -60,7 +60,7 @@ function(add_example_executable EXAMPLE_NAME FILE_NAME) - endforeach() - #Do not build any WMMA examples if gfx11 targets are not on the list - foreach(source IN LISTS FILE_NAME) -- if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "_wmma") -+ if(NOT GPU_TARGETS MATCHES "gfx11" AND NOT GPU_TARGETS MATCHES "gfx12" AND source MATCHES "_wmma") - message("removing wmma example ${source} ") - list(REMOVE_ITEM FILE_NAME "${source}") - endif() -@@ -134,7 +134,7 @@ function(add_example_executable_no_testing EXAMPLE_NAME FILE_NAME) - endforeach() - #Do not build any WMMA examples if gfx11 targets are not on the list - foreach(source IN LISTS FILE_NAME) -- if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "_wmma") -+ if(NOT GPU_TARGETS MATCHES "gfx11" AND NOT GPU_TARGETS MATCHES "gfx12" AND source MATCHES "_wmma") - message("removing wmma example ${source} ") - list(REMOVE_ITEM FILE_NAME "${source}") - endif() -diff --git a/include/ck/ck.hpp b/include/ck/ck.hpp -index 55f562061..69a7abf62 100644 ---- a/include/ck/ck.hpp -+++ b/include/ck/ck.hpp -@@ -69,6 +69,9 @@ CK_DECLARE_ENV_VAR_BOOL(CK_LOGGING) - #if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) - #define __gfx11__ - #endif -+#if defined(__gfx1200__) || defined(__gfx1201__) -+#define __gfx12__ -+#endif - - // buffer resource - #ifndef __HIP_DEVICE_COMPILE__ // for host code -@@ -77,7 +80,7 @@ CK_DECLARE_ENV_VAR_BOOL(CK_LOGGING) - #define CK_BUFFER_RESOURCE_3RD_DWORD 0x00020000 - #elif defined(__gfx103__) - #define CK_BUFFER_RESOURCE_3RD_DWORD 0x31014000 --#elif defined(__gfx11__) -+#elif defined(__gfx11__) || defined(__gfx12__) - #define CK_BUFFER_RESOURCE_3RD_DWORD 0x31004000 - #endif - -@@ -89,7 +92,7 @@ CK_DECLARE_ENV_VAR_BOOL(CK_LOGGING) - #define CK_USE_AMD_V_FMAC_F32 - #define CK_USE_AMD_V_DOT2_F32_F16 - #define CK_USE_AMD_V_DOT4_I32_I8 --#elif defined(__gfx11__) -+#elif defined(__gfx11__) || defined(__gfx12__) - #define CK_USE_AMD_V_FMAC_F32 - #define CK_USE_AMD_V_DOT2_F32_F16 - #define CK_USE_AMD_V_DOT4_I32_I8_GFX11 -@@ -110,13 +113,6 @@ CK_DECLARE_ENV_VAR_BOOL(CK_LOGGING) - #define CK_USE_AMD_MFMA_GFX940 - #endif - --// WMMA instruction --#ifndef __HIP_DEVICE_COMPILE__ // for host code --#define CK_USE_AMD_WMMA --#elif defined(__gfx11__) // for GPU code --#define CK_USE_AMD_WMMA --#endif -- - // buffer load - #define CK_USE_AMD_BUFFER_LOAD 1 - -diff --git a/include/ck/host_utility/device_prop.hpp b/include/ck/host_utility/device_prop.hpp -index 116bb3ea0..83af2efe8 100644 ---- a/include/ck/host_utility/device_prop.hpp -+++ b/include/ck/host_utility/device_prop.hpp -@@ -84,4 +84,9 @@ inline bool is_gfx11_supported() - ck::get_device_name() == "gfx1102" || ck::get_device_name() == "gfx1103"; - } - -+inline bool is_gfx12_supported() -+{ -+ return ck::get_device_name() == "gfx1200" || ck::get_device_name() == "gfx1201"; -+} -+ - } // namespace ck -diff --git a/include/ck/tensor_operation/gpu/block/blockwise_gemm_wmma.hpp b/include/ck/tensor_operation/gpu/block/blockwise_gemm_wmma.hpp -index f8ee283c6..7eb7d42eb 100644 ---- a/include/ck/tensor_operation/gpu/block/blockwise_gemm_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/block/blockwise_gemm_wmma.hpp -@@ -13,6 +13,504 @@ - - namespace ck { - -+#ifdef __gfx12__ -+template -+/* Option: Read from LDS, big buffer hold all threads required data -+ * Source -+ * A: K0PerBlock x MPerBlock x K1 -+ * B: K0PerBlock x NPerBlock x K1 -+ * Destination -+ * C, non-transpose -+ * thread level: MRepeat x NRepeat x MAccVgprs -+ * block level: MRepeat x MWave x MSubGroup x NRepeat x NWave x NThreadPerSubGroup x MAccVgprs -+ * KPACK == WMMA_K = 16 -+ * -+ * Option: Read from VMEM, small buffer hold each thread own required data (Skip LDS) -+ * Source: -+ * A(if skip LDS): MRepeat x KPack -+ * B(if skip LDS): NRepeat x KPack -+ * Destination -+ * C, non-transpose -+ * block level: MRepeat x MWave x MSubGroup x NRepeat x NWave x NThreadPerSubGroup x MAccVgprs -+ */ -+struct BlockwiseGemmWMMA -+{ -+ static constexpr auto I0 = Number<0>{}; -+ static constexpr auto I1 = Number<1>{}; -+ static constexpr auto I2 = Number<2>{}; -+ static constexpr auto I3 = Number<3>{}; -+ static constexpr auto I4 = Number<4>{}; -+ static constexpr auto I5 = Number<5>{}; -+ static constexpr auto WmmaK = Number<16>{}; -+ -+ using ThisThreadBlock = ThisThreadBlock; -+ -+ // Hardcode of WaveSize, since current HIP Runtime(5.4.0-10984) could not return correct one. -+ static constexpr index_t WaveSize = 32; -+ -+ // When use LDS, each Row(16 consecutive lanes) read whole data from source buffer -+ // When not use LDS, each Row read half of whole data from source buffer, exchange the data via -+ // permutation -+ static constexpr index_t A_KRow = 2; -+ static constexpr index_t B_KRow = 2; -+ -+ static constexpr index_t A_K1 = ABlockDesc{}.GetLength(I5); -+ static constexpr index_t B_K1 = BBlockDesc{}.GetLength(I5); -+ -+ static constexpr auto wmma_gemm = -+ WmmaGemm{}; -+ -+ static constexpr index_t MWaves = MPerBlock / (MRepeat * MPerWMMA); -+ static constexpr index_t NWaves = NPerBlock / (NRepeat * NPerWMMA); -+ -+ StaticBufferTupleOfVector -+ c_thread_buf_; -+ -+ __host__ __device__ constexpr auto& GetCThreadBuffer() { return c_thread_buf_; } -+ -+ __device__ static auto GetWaveIdx() -+ { -+ const index_t thread_id = ThisThreadBlock::GetThreadId(); -+ -+ constexpr auto threadid_to_wave_idx_adaptor = make_single_stage_tensor_adaptor( -+ make_tuple(make_merge_transform(make_tuple(MWaves, NWaves, WaveSize))), -+ make_tuple(Sequence<0, 1, 2>{}), -+ make_tuple(Sequence<0>{})); -+ -+ return threadid_to_wave_idx_adaptor.CalculateBottomIndex(make_multi_index(thread_id)); -+ } -+ -+ // Default, Block buffer in LDS, thread level offset enabled -+ __device__ static auto CalculateAThreadOriginDataIndex() -+ { -+ if constexpr(AEnableLds) -+ { -+ const auto wave_idx = GetWaveIdx(); -+ const auto waveId_m = wave_idx[I0]; -+ const auto WMMA_a_idx = wmma_gemm.CalculateAThreadOriginDataIndex(); -+ -+ // |KRepeat |MRepeat|MWave |KRow |MLane |KPack -+ return make_tuple(0, 0, waveId_m, wmma_gemm.GetSubGroupId(), WMMA_a_idx, 0); -+ } -+ else -+ { -+ return make_tuple(0, 0, 0, 0, 0, 0); -+ } -+ } -+ -+ __device__ static auto CalculateBThreadOriginDataIndex() -+ { -+ if constexpr(BEnableLds) -+ { -+ const auto wave_idx = GetWaveIdx(); -+ const auto waveId_n = wave_idx[I1]; -+ const auto WMMA_b_idx = wmma_gemm.CalculateBThreadOriginDataIndex(); -+ -+ // |KRepeat |NRepeat|Nwave |KRow |NLane |KPack -+ return make_tuple(0, 0, waveId_n, wmma_gemm.GetSubGroupId(), WMMA_b_idx, 0); -+ } -+ else -+ { -+ return make_tuple(0, 0, 0, 0, 0, 0); -+ } -+ } -+ -+ template -+ __device__ static auto CalculateCThreadOriginDataIndex(Number, Number) -+ { -+ const auto wave_idx = GetWaveIdx(); -+ -+ const auto waveId_m = wave_idx[I0]; -+ const auto waveId_n = wave_idx[I1]; -+ -+ const auto blk_idx = wmma_gemm.GetBeginOfThreadBlk(); -+ -+ constexpr auto mrepeat_mwave_mperWMMA_to_m_adaptor = make_single_stage_tensor_adaptor( -+ make_tuple(make_unmerge_transform(make_tuple(MRepeat, MWaves, MPerWMMA))), -+ make_tuple(Sequence<0>{}), -+ make_tuple(Sequence<0, 1, 2>{})); -+ -+ constexpr auto nrepeat_nwave_nperWMMA_to_n_adaptor = make_single_stage_tensor_adaptor( -+ make_tuple(make_unmerge_transform(make_tuple(NRepeat, NWaves, NPerWMMA))), -+ make_tuple(Sequence<0>{}), -+ make_tuple(Sequence<0, 1, 2>{})); -+ -+ const index_t c_thread_m = mrepeat_mwave_mperWMMA_to_m_adaptor.CalculateBottomIndex( -+ make_tuple(m0, waveId_m, blk_idx[I0]))[I0]; -+ const index_t c_thread_n = nrepeat_nwave_nperWMMA_to_n_adaptor.CalculateBottomIndex( -+ make_tuple(n0, waveId_n, blk_idx[I1]))[I0]; -+ -+ return make_tuple(c_thread_m, c_thread_n); -+ } -+ -+ template -+ __device__ static auto CalculateCThreadOriginDataIndex7D(Number, Number) -+ { -+ const auto wave_idx = GetWaveIdx(); -+ -+ const auto waveId_m = wave_idx[I0]; -+ const auto waveId_n = wave_idx[I1]; -+ -+ const auto blk_idx = wmma_gemm.GetBeginOfThreadBlk3D(); -+ -+ return make_tuple( -+ Number{}, waveId_m, blk_idx[I0], Number{}, waveId_n, blk_idx[I1], blk_idx[I2]); -+ } -+ -+ using Tuple6 = decltype(CalculateAThreadOriginDataIndex()); -+ __host__ __device__ BlockwiseGemmWMMA(Tuple6 a_origin = CalculateAThreadOriginDataIndex(), -+ Tuple6 b_origin = CalculateBThreadOriginDataIndex()) -+ : a_thread_copy_(a_origin), b_thread_copy_(b_origin) -+ { -+ static_assert(ABlockDesc::IsKnownAtCompileTime() && BBlockDesc::IsKnownAtCompileTime(), -+ "wrong! Desc should be known at compile-time"); -+ -+ static_assert(ThisThreadBlock::GetNumOfThread() == MWaves * NWaves * WaveSize, -+ "ThisThreadBlock::GetNumOfThread() != MWaves * NWaves * WaveSize\n"); -+ -+ static_assert(MPerBlock % (MPerWMMA * MRepeat) == 0 && -+ NPerBlock % (NPerWMMA * NRepeat) == 0, -+ "wrong!"); -+ } -+ -+ // transposed WMMA output C' = B' * A' -+ __host__ __device__ static constexpr auto -+ GetCThreadDescriptor_MRepeat_MWave_MThreadPerSubGroup_NRepeat_NWave_NSubGroup_NAccVgprs() -+ { -+ constexpr auto c_msubgroup_nthreadpersubgroup_maccvgprs_tblk_lens = -+ wmma_gemm.GetCMSubGroupNThreadPerSubGroupMAccVgprsThreadBlkLengths(); -+ -+ constexpr auto NAccVgprs = c_msubgroup_nthreadpersubgroup_maccvgprs_tblk_lens[I2]; -+ -+ return make_naive_tensor_descriptor_packed( -+ // |MRepeat |MWave |MSubGroup |NRepeat |NWave -+ // |NThreadPerSubGroup |MAccVgprs -+ make_tuple(Number{}, I1, I1, Number{}, I1, I1, NAccVgprs)); -+ } -+ -+ // Thread level, register decriptor. Vector-write -+ __host__ __device__ static constexpr auto -+ GetCThreadDescriptor_MRepeat_MWave_MSubGroup_NRepeat_NWave_NThreadPerSubGroup_MAccVgprs() -+ { -+ constexpr auto c_msubgroup_nthreadpersubgroup_maccvgprs_tblk_lens = -+ wmma_gemm.GetCMSubGroupNThreadPerSubGroupMAccVgprsThreadBlkLengths(); -+ -+ constexpr auto MAccVgprs = c_msubgroup_nthreadpersubgroup_maccvgprs_tblk_lens[I2]; -+ constexpr auto AccStride = c_msubgroup_nthreadpersubgroup_maccvgprs_tblk_lens[I3]; -+ return make_naive_tensor_descriptor( -+ // |MRepeat |MWave |MSubGroup |NRepeat |NWave -+ // |NThreadPerSubGroup |MAccVgprs -+ make_tuple(Number{}, I1, I1, Number{}, I1, I1, MAccVgprs), -+ make_tuple(Number{} * MAccVgprs * AccStride, -+ Number{} * MAccVgprs * AccStride, -+ Number{} * MAccVgprs * AccStride, -+ MAccVgprs * AccStride, -+ MAccVgprs * AccStride, -+ MAccVgprs * AccStride, -+ AccStride)); -+ } -+ -+ template -+ __host__ __device__ static constexpr auto -+ MakeCGridDescriptor_MBlockxRepeat_MWave_MSubGroup_NBlockxRepeat_NWave_NThreadPerSubGroup_MAccVgprs( -+ const CGridDesc_M_N& c_grid_desc_m_n) -+ { -+ const auto M = c_grid_desc_m_n.GetLength(I0); -+ const auto N = c_grid_desc_m_n.GetLength(I1); -+ -+ const auto c_grid_desc_mblockxrepeat_mwave_mperwmma_nblockxrepeat_nwave_nperwmma = -+ transform_tensor_descriptor( -+ c_grid_desc_m_n, -+ make_tuple( -+ make_unmerge_transform(make_tuple(M / (MWaves * MPerWMMA), MWaves, MPerWMMA)), -+ make_unmerge_transform(make_tuple(N / (NWaves * NPerWMMA), NWaves, NPerWMMA))), -+ make_tuple(Sequence<0>{}, Sequence<1>{}), -+ make_tuple(Sequence<0, 1, 2>{}, Sequence<3, 4, 5>{})); -+ -+ return wmma_gemm -+ .MakeCDesc_MBlockxRepeat_MWave_MSubGroup_NBlockxRepeat_NWave_NThreadPerSubGroup_MAccVgprs( -+ c_grid_desc_mblockxrepeat_mwave_mperwmma_nblockxrepeat_nwave_nperwmma); -+ } -+ -+ // transposed WMMA output C' = B' * A' -+ __host__ __device__ static constexpr auto -+ GetCBlockDescriptor_MRepeat_MWave_MThreadPerSubGroup_NRepeat_NWave_NSubGroup_NAccVgprs() -+ { -+ constexpr auto c_block_desc_mrepeat_mwave_mperwmma_nrepeat_nwave_nperwmma = -+ make_naive_tensor_descriptor_packed(make_tuple(Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{})); -+ -+ return wmma_gemm -+ .MakeCDesc_MBlockxRepeat_MWave_MThreadPerSubGroup_NBlockxRepeat_NWave_NSubGroup_NAccVgprs( -+ c_block_desc_mrepeat_mwave_mperwmma_nrepeat_nwave_nperwmma); -+ } -+ -+ // Provide dimension size -+ __host__ __device__ static constexpr auto -+ GetCBlockDescriptor_MRepeat_MWave_MSubGroup_NRepeat_NWave_NThreadPerSubGroup_MAccVgprs() -+ { -+ constexpr auto c_block_desc_mrepeat_mwave_mperwmma_nrepeat_nwave_nperwmma = -+ make_naive_tensor_descriptor_packed(make_tuple(Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{})); -+ -+ return wmma_gemm -+ .MakeCDesc_MBlockxRepeat_MWave_MSubGroup_NBlockxRepeat_NWave_NThreadPerSubGroup_MAccVgprs( -+ c_block_desc_mrepeat_mwave_mperwmma_nrepeat_nwave_nperwmma); -+ } -+ -+ // Describe how data allocated in thread copy src buffer -+ // M0_M1_M2 = MRepeat_MWave_MPerWmma, N0_N1_N2 = NRepeat_NWave_NPerWmma -+ static constexpr ABlockDesc a_block_desc_k0_m0_m1_m2_k1; -+ static constexpr BBlockDesc b_block_desc_k0_n0_n1_n2_k1; -+ -+ template -+ __device__ void Run(const ABlockBuffer& a_block_buf, -+ const BBlockBuffer& b_block_buf, -+ CThreadBuffer& c_thread_buf) const -+ { -+ auto a_thread_buf = make_static_buffer( -+ a_thread_desc_.GetElementSpaceSize()); -+ auto b_thread_buf = make_static_buffer( -+ b_thread_desc_.GetElementSpaceSize()); -+ -+ static_assert(KPack % (A_K1 * A_KRow) == 0, ""); -+ static_assert(KPack % (B_K1 * B_KRow) == 0, ""); -+ -+ // basic intrinsic to determine loopover direction -+ if constexpr(MRepeat < NRepeat) -+ { -+ static_for<0, KPerBlock / KPack, 1>{}( -+ [&](auto k) { // k=0,1,2 instead of k=0,kpack*1, ... -+ static_for<0, MRepeat, 1>{}([&](auto m0) { -+ // read A -+ a_thread_copy_.Run( -+ a_block_desc_k0_m0_m1_m2_k1, -+ make_tuple(Number{}, m0, I0, I0, I0, I0), -+ a_block_buf, -+ a_thread_desc_, -+ make_tuple(I0, m0, I0, I0, I0, I0), -+ a_thread_buf); -+ -+ static_for<0, NRepeat, 1>{}([&](auto n0) { -+ // read B -+ b_thread_copy_.Run( -+ b_block_desc_k0_n0_n1_n2_k1, -+ make_tuple(Number{}, n0, I0, I0, I0, I0), -+ b_block_buf, -+ b_thread_desc_, -+ make_tuple(I0, n0, I0, I0, I0, I0), -+ b_thread_buf); -+ -+ vector_type a_thread_vec; -+ vector_type b_thread_vec; -+ -+ static_for<0, KPack / A_KRow, 1>{}([&](auto i) { -+ a_thread_vec.template AsType()(i) = -+ a_thread_buf[Number{}]; -+ }); -+ -+ static_for<0, KPack / B_KRow, 1>{}([&](auto i) { -+ b_thread_vec.template AsType()(i) = -+ b_thread_buf[Number{}]; -+ }); -+ -+ using wmma_input_type_a = -+ typename vector_type::type; -+ using wmma_input_type_b = -+ typename vector_type::type; -+ -+ constexpr index_t c_offset = -+ c_thread_desc_.CalculateOffset(make_tuple(m0, n0, 0)); -+ -+ wmma_gemm.template Run( -+ a_thread_vec.template AsType(), -+ b_thread_vec.template AsType(), -+ c_thread_buf.GetVectorTypeReference(Number{})); -+ }); -+ }); -+ }); -+ } -+ else -+ { -+ static_for<0, NRepeat, 1>{}([&](auto n0) { -+ static_for<0, MRepeat, 1>{}([&](auto m0) { -+ static_for<0, KPerBlock / KPack, 1>{}([&](auto k) { // k=0,1,2 instead of -+ // k=0,kpack*1, .. -+ // read B -+ b_thread_copy_.Run( -+ b_block_desc_k0_n0_n1_n2_k1, -+ make_tuple(Number{}, n0, I0, I0, I0, I0), -+ b_block_buf, -+ b_thread_desc_, -+ make_tuple(I0, n0, I0, I0, I0, I0), -+ b_thread_buf); -+ // read A -+ a_thread_copy_.Run( -+ a_block_desc_k0_m0_m1_m2_k1, -+ make_tuple(Number{}, m0, I0, I0, I0, I0), -+ a_block_buf, -+ a_thread_desc_, -+ make_tuple(I0, m0, I0, I0, I0, I0), -+ a_thread_buf); -+ -+ vector_type a_thread_vec; -+ vector_type b_thread_vec; -+ -+ static_for<0, KPack / A_KRow, 1>{}([&](auto i) { -+ a_thread_vec.template AsType()(i) = -+ a_thread_buf[Number{}]; -+ }); -+ -+ static_for<0, KPack / B_KRow, 1>{}([&](auto i) { -+ b_thread_vec.template AsType()(i) = -+ b_thread_buf[Number{}]; -+ }); -+ -+ using wmma_input_type_a = -+ typename vector_type::type; -+ using wmma_input_type_b = -+ typename vector_type::type; -+ -+ constexpr index_t c_offset = -+ c_thread_desc_.CalculateOffset(make_tuple(m0, n0, 0)); -+ -+ wmma_gemm.template Run( -+ a_thread_vec.template AsType(), -+ b_thread_vec.template AsType(), -+ c_thread_buf.GetVectorTypeReference(Number{})); -+ }); -+ }); -+ }); -+ } -+ } -+ -+ protected: -+ static constexpr auto a_thread_desc_ = make_naive_tensor_descriptor( -+ make_tuple(Number{}, Number{}, I1, I1, I1, Number{}), -+ make_tuple(Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number<1>{})); -+ -+ static constexpr auto b_thread_desc_ = make_naive_tensor_descriptor( -+ make_tuple(Number{}, Number{}, I1, I1, I1, Number{}), -+ make_tuple(Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number{}, -+ Number<1>{})); -+ -+ // C[M, N, NumRegWMMA] -+ static constexpr auto c_thread_desc_ = make_naive_tensor_descriptor_packed( -+ make_tuple(Number{}, Number{}, wmma_gemm.GetRegSizePerWmma())); -+ -+ template -+ struct AThreadCopySelector; -+ -+ template <> -+ struct AThreadCopySelector -+ { -+ using type = -+ ThreadwiseTensorSliceTransfer_v4, -+ Sequence<0, 1, 2, 3, 4, 5>, -+ 5, -+ A_K1, -+ A_K1>; -+ }; -+ -+ template <> -+ struct AThreadCopySelector -+ { -+ using type = ThreadwiseTensorSliceTransfer_StaticToStatic_IntraRow< -+ FloatA, -+ FloatA, -+ decltype(a_block_desc_k0_m0_m1_m2_k1), -+ decltype(a_thread_desc_), -+ tensor_operation::element_wise::PassThrough, -+ Sequence, -+ Sequence<0, 1, 2, 3, 4, 5>, -+ 5, -+ A_K1, -+ false>; -+ }; -+ -+ template -+ struct BThreadCopySelector; -+ -+ template <> -+ struct BThreadCopySelector -+ { -+ using type = -+ ThreadwiseTensorSliceTransfer_v4, -+ Sequence<0, 1, 2, 3, 4, 5>, -+ 5, -+ B_K1, -+ B_K1>; -+ }; -+ -+ template <> -+ struct BThreadCopySelector -+ { -+ using type = ThreadwiseTensorSliceTransfer_StaticToStatic_IntraRow< -+ FloatB, -+ FloatB, -+ decltype(b_block_desc_k0_n0_n1_n2_k1), -+ decltype(b_thread_desc_), -+ tensor_operation::element_wise::PassThrough, -+ Sequence, -+ Sequence<0, 1, 2, 3, 4, 5>, -+ 5, -+ B_K1, -+ false>; -+ }; -+ -+ typename AThreadCopySelector::type a_thread_copy_; -+ typename BThreadCopySelector::type b_thread_copy_; -+}; -+#else - template ::type a_thread_copy_; - typename BThreadCopySelector::type b_thread_copy_; - }; -+#endif - - } // namespace ck -diff --git a/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp b/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp -index e5e6245cb..1f7d50429 100644 ---- a/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp -+++ b/include/ck/tensor_operation/gpu/block/blockwise_gemm_xdlops.hpp -@@ -488,7 +488,14 @@ struct BlockwiseGemmXdlopsInterwave_k0mk1_k0nk1_m0n0m1n1m2m3m4n2_v1 - // sync point. - if constexpr(k.value != 0 || KPerInnerLoop == KPerThread) - { -+#ifdef __gfx12__ -+ asm volatile("\ -+ s_barrier_signal -1 \n \ -+ s_barrier_wait -1 \ -+ " ::); -+#else - asm volatile("s_barrier" ::); -+#endif - __builtin_amdgcn_sched_barrier(0); - } - static_for<0, KPerInnerLoop, KPack>{}([&](auto k_) { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp -index a15759559..ab3f3856a 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_batched_contraction_multiple_d_wmma_cshuffle.hpp -@@ -133,8 +133,13 @@ struct DeviceBatchedContractionMultipleD_Wmma_CShuffle - static constexpr auto NWaves = NPerBlock / (NRepeat * NPerWmma); - static constexpr auto WmmaK = K1 == 16 ? 32 : 16; - -- static constexpr auto AEnableLds_auto = NWaves == 1 ? false : true; -- static constexpr auto BEnableLds_auto = MWaves == 1 ? false : true; -+ static constexpr auto MaxVectorLoadA = K1 * sizeof(ADataType) == 16 ? true : false; -+ static constexpr auto MaxVectorLoadB = K1 * sizeof(BDataType) == 16 ? true : false; -+ -+ static constexpr auto AEnableLds_auto = -+ (NWaves == 1 && (MaxVectorLoadA || MRepeat == 1)) ? false : true; -+ static constexpr auto BEnableLds_auto = -+ (MWaves == 1 && (MaxVectorLoadB || NRepeat == 1)) ? false : true; - - // If true, LDS is used unconditionally - static constexpr auto AEnableLds_manu = false; -@@ -829,7 +834,7 @@ struct DeviceBatchedContractionMultipleD_Wmma_CShuffle - - static bool IsSupportedArgument(const Argument& arg) - { -- if(ck::is_gfx11_supported()) -+ if(ck::is_gfx11_supported() || ck::is_gfx12_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { -@@ -869,11 +874,15 @@ struct DeviceBatchedContractionMultipleD_Wmma_CShuffle - } - else - { -- if(!(arg.a_kz_stride_ == 1 && -- arg.a_grid_desc_.GetLength(I2) % ABlockTransferSrcScalarPerVector == 0)) -+ if(!(arg.a_kz_stride_ == 1)) - { -- printf("DeviceOp: Vector Access A-k check failure\n"); -- return false; -+ index_t LastK = -+ AEnableLds ? arg.a_grid_desc_.GetLength(I2) : arg.a_grid_desc_.GetLength(I6); -+ if(LastK % ABlockTransferSrcScalarPerVector == 0) -+ { -+ printf("DeviceOp: Vector Access A-k check failure\n"); -+ return false; -+ } - } - } - -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_multiple_d_dl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_multiple_d_dl.hpp -index 8fd14afc0..1b487502f 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_multiple_d_dl.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_batched_gemm_multiple_d_dl.hpp -@@ -70,8 +70,9 @@ __global__ void - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch, - const Block2CTileMap block_2_ctile_map) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -- defined(__gfx90a__) || defined(__gfx94__) || defined(__gfx103__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -+ defined(__gfx90a__) || defined(__gfx94__) || defined(__gfx103__) || defined(__gfx11__) || \ -+ defined(__gfx12__)) - - const index_t num_blocks_per_batch = - __builtin_amdgcn_readfirstlane(get_grid_size() / batch_count); -@@ -648,7 +649,7 @@ struct DeviceBatchedGemmMultipleD_Dl : public DeviceBatchedGemmMultiD || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_contraction_multiple_d_xdl_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_contraction_multiple_d_xdl_cshuffle.hpp -index 9d5b74be6..017d28641 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_contraction_multiple_d_xdl_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_contraction_multiple_d_xdl_cshuffle.hpp -@@ -601,9 +601,7 @@ struct DeviceContractionMultipleD_Xdl_CShuffle - return false; - } - -- if(ck::get_device_name() != "gfx90a" && ck::get_device_name() != "gfx940" && -- ck::get_device_name() != "gfx941" && ck::get_device_name() != "gfx942" && -- std::is_same::value) -+ if(!ck::is_lds_direct_load_supported() && std::is_same::value) - { - return false; - } -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_convnd_bwd_data_nwc_kxc_nwk_dl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_convnd_bwd_data_nwc_kxc_nwk_dl.hpp -index b84e18130..1edae33be 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_convnd_bwd_data_nwc_kxc_nwk_dl.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_convnd_bwd_data_nwc_kxc_nwk_dl.hpp -@@ -1393,7 +1393,7 @@ struct DeviceConvNdBwdDataNwcKxcNwk_Dl - { - // check device - if(!(ck::get_device_name() == "gfx906" || ck::is_gfx103_supported() || -- ck::is_gfx11_supported())) -+ ck::is_gfx11_supported() || ck::is_gfx12_supported())) - { - return false; - } -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_fpAintB_gemm_wmma.hpp b/include/ck/tensor_operation/gpu/device/impl/device_fpAintB_gemm_wmma.hpp -index bf96324d0..553143e28 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_fpAintB_gemm_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_fpAintB_gemm_wmma.hpp -@@ -509,7 +509,7 @@ struct DeviceFpAintBGemm_Wmma_CShuffle : public DeviceGemm_dequantB || is_same_v || - is_same_v)) -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_gemm_dl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_gemm_dl.hpp -index b1784b385..eb0fb55f5 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_gemm_dl.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_gemm_dl.hpp -@@ -536,7 +536,7 @@ struct DeviceGemmDl : public DeviceGemm || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_gemm_wmma.hpp b/include/ck/tensor_operation/gpu/device/impl/device_gemm_wmma.hpp -index 93ab8a7e1..a7cc546f5 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_gemm_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_gemm_wmma.hpp -@@ -84,14 +84,21 @@ struct DeviceGemmWmma_CShuffle : public DeviceGemm{}; - -- static constexpr auto MWaves = MPerBlock / (MRepeat * MPerWmma); -- static constexpr auto NWaves = NPerBlock / (NRepeat * NPerWmma); -- static constexpr auto WmmaK = K1 == 16 ? 32 : 16; -- -- static constexpr auto AEnableLds_auto = -- (NWaves == 1 && is_same::value) ? false : true; -+ static constexpr auto MWaves = MPerBlock / (MRepeat * MPerWmma); -+ static constexpr auto NWaves = NPerBlock / (NRepeat * NPerWmma); -+ static constexpr auto WmmaK = K1 == 16 ? 32 : 16; -+ static constexpr auto MaxVectorLoadA = K1 * sizeof(ADataType) == 16 ? true : false; -+ static constexpr auto MaxVectorLoadB = K1 * sizeof(BDataType) == 16 ? true : false; -+ -+ static constexpr auto AEnableLds_auto = (NWaves == 1 && (MaxVectorLoadA || MRepeat == 1) && -+ is_same::value) -+ ? false -+ : true; - static constexpr auto BEnableLds_auto = -- (MWaves == 1 && is_same::value) ? false : true; -+ (MWaves == 1 && (MaxVectorLoadB || NRepeat == 1) && -+ is_same::value) -+ ? false -+ : true; - - // If true, LDS is used unconditionally - static constexpr auto AEnableLds_manu = false; -@@ -443,7 +450,7 @@ struct DeviceGemmWmma_CShuffle : public DeviceGemm || is_same_v || - is_same_v)) -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_wmma_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_wmma_cshuffle.hpp -index 6f74838fb..6bb5d431c 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_wmma_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_data_multiple_d_wmma_cshuffle.hpp -@@ -629,7 +629,7 @@ struct DeviceGroupedConvBwdDataMultipleD_Wmma_CShuffle - static bool IsSupportedArgument(const Argument& arg) - { - // check device -- if(ck::is_gfx11_supported()) -+ if(ck::is_gfx11_supported() || ck::is_gfx12_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_dl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_dl.hpp -index bd264a3c8..7047e1bda 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_dl.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_dl.hpp -@@ -48,8 +48,9 @@ __global__ void - const Block2CTileMap block_2_ctile_map, - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx103__) || \ -- defined(__gfx90a__) || defined(__gfx908__) || defined(__gfx94__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx103__) || \ -+ defined(__gfx90a__) || defined(__gfx908__) || defined(__gfx94__) || defined(__gfx11__) || \ -+ defined(__gfx12__)) - const index_t num_blocks_per_batch = - __builtin_amdgcn_readfirstlane(get_grid_size() / batch_count); - const index_t g_idx = __builtin_amdgcn_readfirstlane(get_block_1d_id() / num_blocks_per_batch); -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_wmma_cshuffle.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_wmma_cshuffle.hpp -index 211185dfb..5738be0fb 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_wmma_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_bwd_weight_wmma_cshuffle.hpp -@@ -692,7 +692,7 @@ struct DeviceGroupedConvBwdWeight_Wmma_CShuffle - static bool IsSupportedArgument(const Argument& arg) - { - // check device -- if(ck::is_gfx11_supported()) -+ if(ck::is_gfx11_supported() || ck::is_gfx12_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_multiple_d_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_multiple_d_nhwc_kyxc_nhwk.hpp -index 7cfbd8a8f..5d5a9de7d 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_multiple_d_nhwc_kyxc_nhwk.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_multiple_d_nhwc_kyxc_nhwk.hpp -@@ -90,8 +90,9 @@ __global__ void - const Block2CTileMap block_2_ctile_map, - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx103__) || \ -- defined(__gfx90a__) || defined(__gfx908__) || defined(__gfx94__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx103__) || \ -+ defined(__gfx90a__) || defined(__gfx908__) || defined(__gfx94__) || defined(__gfx11__) || \ -+ defined(__gfx12__)) - // offset base pointer for each work-group - const index_t num_blocks_per_batch = - __builtin_amdgcn_readfirstlane(get_grid_size() / batch_count); -@@ -666,7 +667,7 @@ struct DeviceGroupedConvFwdDlMultipleD_NHWC_KYXC_NHWK - - // check device - if(!(ck::get_device_name() == "gfx906" || ck::is_xdl_supported() || -- ck::is_gfx103_supported() || ck::is_gfx11_supported())) -+ ck::is_gfx103_supported() || ck::is_gfx11_supported() || ck::is_gfx12_supported())) - { - return false; - } -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_nhwc_kyxc_nhwk.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_nhwc_kyxc_nhwk.hpp -index 6a4d97d7d..c65370b51 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_nhwc_kyxc_nhwk.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_conv_fwd_dl_nhwc_kyxc_nhwk.hpp -@@ -107,7 +107,7 @@ __global__ void - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch) - { - #if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx103__) || \ -- defined(__gfx11__)) -+ defined(__gfx11__) || defined(__gfx12__)) - // offset base pointer for each work-group - const index_t num_blocks_per_batch = - __builtin_amdgcn_readfirstlane(get_grid_size() / batch_count); -@@ -602,7 +602,7 @@ struct DeviceGroupedConvFwdDl_NHWC_KYXC_NHWK : public DeviceGroupedConvFwd || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_grouped_gemm_multiple_d_dl.hpp b/include/ck/tensor_operation/gpu/device/impl/device_grouped_gemm_multiple_d_dl.hpp -index ac392cddc..060a16d1e 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_grouped_gemm_multiple_d_dl.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_grouped_gemm_multiple_d_dl.hpp -@@ -39,8 +39,9 @@ __global__ void - const BElementwiseOperation b_element_op, - const CDEElementwiseOperation cde_element_op) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -- defined(__gfx90a__) || defined(__gfx103__) || defined(__gfx11__) || defined(__gfx94__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -+ defined(__gfx90a__) || defined(__gfx103__) || defined(__gfx11__) || defined(__gfx94__) || \ -+ defined(__gfx12__)) - __shared__ char p_shared[GridwiseGemm::GetSharedMemoryNumberOfByte()]; - - const index_t block_id = get_block_1d_id(); -@@ -673,7 +674,7 @@ struct DeviceGroupedGemmMultipleD_Dl : public DeviceGroupedGemm || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp b/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp -index 4e14ed3a5..cc88c1a10 100644 ---- a/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/device/impl/device_multi_query_attention_forward_wmma.hpp -@@ -60,7 +60,7 @@ __global__ void - bool input_permute, - bool output_permute) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - - // clang-format off - // *************************************************** -@@ -165,6 +165,7 @@ __global__ void - ignore = O; - ignore = G0; - ignore = G1; -+ ignore = alpha; - ignore = input_permute; - ignore = output_permute; - #endif // end of if (defined(__gfx11__)) -@@ -594,7 +595,7 @@ struct DeviceMultiQueryAttentionForward_Wmma - - static bool IsSupportedArgument(const RawArg& arg) - { -- if(ck::is_gfx11_supported()) -+ if(ck::is_gfx11_supported() || ck::is_gfx12_supported()) - { - if constexpr(!(is_same_v || is_same_v)) - { -diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_batched_gemm_softmax_gemm_wmma_cshuffle.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_batched_gemm_softmax_gemm_wmma_cshuffle.hpp -index 16717ff81..1754e07e6 100644 ---- a/include/ck/tensor_operation/gpu/grid/gridwise_batched_gemm_softmax_gemm_wmma_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/grid/gridwise_batched_gemm_softmax_gemm_wmma_cshuffle.hpp -@@ -371,12 +371,16 @@ struct GridwiseBatchedGemmSoftmaxGemm_Wmma - if constexpr(B0EnableLds) - { - // BK0_L_BK1 -> BK0_LRepeat_Lwaves_LPerWmma_BK1 -- constexpr auto B_K0 = B0BlockDesc_{}.GetLength(I0); -- constexpr auto B_K1 = B0BlockDesc_{}.GetLength(I2); -+ constexpr auto B_K0 = B0BlockDesc_{}.GetLength(I0); -+ constexpr auto B_K1 = B0BlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto B_KRow = I2; -+#else - constexpr auto B_KRow = I1; -+#endif - return transform_tensor_descriptor( - B0BlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -428,12 +432,16 @@ struct GridwiseBatchedGemmSoftmaxGemm_Wmma - if constexpr(B1EnableLds) - { - // BL0_N_BL1 -> BL0_NRepeat_Nwaves_NPerWmma_BL1 -- constexpr auto B_L0 = B1BlockDesc_{}.GetLength(I0); -- constexpr auto B_L1 = B1BlockDesc_{}.GetLength(I2); -+ constexpr auto B_L0 = B1BlockDesc_{}.GetLength(I0); -+ constexpr auto B_L1 = B1BlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto B_LRow = I2; -+#else - constexpr auto B_LRow = I1; -+#endif - return transform_tensor_descriptor( - B1BlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, B_LRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, B_LRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_fpAintB_gemm_wmma.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_fpAintB_gemm_wmma.hpp -index 499eb7eb0..21dac6f9e 100644 ---- a/include/ck/tensor_operation/gpu/grid/gridwise_fpAintB_gemm_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/grid/gridwise_fpAintB_gemm_wmma.hpp -@@ -50,7 +50,7 @@ __global__ void - const CElementwiseOperation c_element_op, - const Block2CTileMap block_2_ctile_map) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - __shared__ char p_shared[GridwiseGemm::SharedMemTrait::lds_size]; - - GridwiseGemm::template Run(p_a_grid, -@@ -302,12 +302,16 @@ struct GridwiseFpAintBGemm_Wmma - if constexpr(AEnableLds) - { - // AK0_M_AK1 -> AK0_MRepeat_Mwaves_AKRow_MPerWmma_AK1 -- constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -- constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+ constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -+ constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto A_KRow = I2; -+#else - constexpr auto A_KRow = I1; -+#endif - return transform_tensor_descriptor( - ABlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -360,12 +364,16 @@ struct GridwiseFpAintBGemm_Wmma - if constexpr(BEnableLds) - { - // BK0_N_BK1 -> BK0_NRepeat_Nwaves_NPerWmma_BK1 -- constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -- constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+ constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -+ constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto B_KRow = I2; -+#else - constexpr auto B_KRow = I1; -+#endif - return transform_tensor_descriptor( - BBlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_wmma_cshuffle.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_wmma_cshuffle.hpp -index 82d010a99..fdda649ef 100644 ---- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_wmma_cshuffle.hpp -+++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_multiple_d_wmma_cshuffle.hpp -@@ -54,7 +54,7 @@ __global__ void - const Block2CTileMap block_2_ctile_map, - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - // offset base pointer for each work-group - const index_t num_blocks_per_batch = - __builtin_amdgcn_readfirstlane(get_grid_size() / batch_count); -@@ -147,7 +147,7 @@ __global__ void - const ComputePtrOffsetOfBatch compute_ptr_offset_of_batch, - const Block2CTileMap block_2_etile_map) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - // printf("entry kernel launch"); - __shared__ char p_shared[GridwiseOp::SharedMemTrait::lds_size]; - -@@ -237,7 +237,7 @@ __global__ void - const CDEElementwiseOperation cde_element_op, - const Block2CTileMap block_2_ctile_map) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - __shared__ char p_shared[GridwiseOp::SharedMemTrait::lds_size]; - - GridwiseOp::template Run(p_a_grid, -@@ -375,8 +375,9 @@ struct GridwiseGemmMultipleD_Wmma - } - else - { -+ constexpr auto A_KRow = I2; - constexpr auto KWmmaPerblock = KPerBlock / WmmaK; -- constexpr auto K0PerWmma = WmmaK / 2 / K1; -+ constexpr auto K0PerWmma = WmmaK / A_KRow / K1; - // KWmma->MRepeat->MWave->K0PerWmma->KRow->MPerWmma->K1 Per Thread - return make_naive_tensor_descriptor( - make_tuple(Number{}, -@@ -422,8 +423,9 @@ struct GridwiseGemmMultipleD_Wmma - } - else - { -+ constexpr auto B_KRow = I2; - constexpr auto KWmmaPerblock = KPerBlock / WmmaK; -- constexpr auto K0PerWmma = WmmaK / 2 / K1; -+ constexpr auto K0PerWmma = WmmaK / B_KRow / K1; - // KWmma->NRepeat->MWave->K0PerWmma->KRow->MPerWmma->K1 Per Thread - return make_naive_tensor_descriptor( - make_tuple(Number{}, -@@ -495,12 +497,16 @@ struct GridwiseGemmMultipleD_Wmma - if constexpr(AEnableLds) - { - // AK0_M_AK1 -> AK0_MRepeat_Mwaves_AKRow_MPerWmma_AK1 -- constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -- constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+ constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -+ constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto A_KRow = I2; -+#else - constexpr auto A_KRow = I1; -+#endif - return transform_tensor_descriptor( - ABlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -534,12 +540,16 @@ struct GridwiseGemmMultipleD_Wmma - if constexpr(BEnableLds) - { - // BK0_N_BK1 -> BK0_NRepeat_Nwaves_NPerWmma_BK1 -- constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -- constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+ constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -+ constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto B_KRow = I2; -+#else - constexpr auto B_KRow = I1; -+#endif - return transform_tensor_descriptor( - BBlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -571,15 +581,12 @@ struct GridwiseGemmMultipleD_Wmma - // *Caution Here repeat is shuffle repeat - GetCShuffleBlockDescriptor_MShRepeat_MPerShRepeat_NShRepeat_NPerShRepeat() - { -- constexpr index_t MWave = MPerBlock / (MRepeat * MPerWmma); -- constexpr index_t NWave = NPerBlock / (NRepeat * NPerWmma); -- - constexpr auto c_shuffle_block_desc_mshrepeat_mpershrepeat_nshrepeat_npershrepeat = - make_naive_tensor_descriptor_packed( - make_tuple(I1, -- Number{}, -+ Number{}, - I1, -- Number{})); -+ Number{})); - - return c_shuffle_block_desc_mshrepeat_mpershrepeat_nshrepeat_npershrepeat; - } -@@ -799,8 +806,9 @@ struct GridwiseGemmMultipleD_Wmma - const auto M = e_grid_desc_m_n.GetLength(I0); - const auto N = e_grid_desc_m_n.GetLength(I1); - -- const auto MBlock = M / MPerBlock; -- const auto NBlock = N / NPerBlock; -+ const auto MBlock = M / MPerBlock; -+ const auto NBlock = N / NPerBlock; -+ - const auto e_grid_desc_mblock_mperblock_nblock_nperblock = transform_tensor_descriptor( - e_grid_desc_m_n, - make_tuple(make_unmerge_transform(make_tuple(MBlock, Number{})), -diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_wmma.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_wmma.hpp -index 8e4117593..4458b9356 100644 ---- a/include/ck/tensor_operation/gpu/grid/gridwise_gemm_wmma.hpp -+++ b/include/ck/tensor_operation/gpu/grid/gridwise_gemm_wmma.hpp -@@ -45,7 +45,7 @@ __global__ void - const CElementwiseOperation c_element_op, - const Block2CTileMap block_2_ctile_map) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx11__) || defined(__gfx12__)) - __shared__ char p_shared[GridwiseGemm::SharedMemTrait::lds_size]; - - GridwiseGemm::template Run(p_a_grid, -@@ -170,8 +170,9 @@ struct GridwiseGemm_Wmma - } - else - { -+ constexpr auto A_KRow = I2; - constexpr auto KWmmaPerblock = KPerBlock / WmmaK; -- constexpr auto K0PerWmma = WmmaK / 2 / K1; -+ constexpr auto K0PerWmma = WmmaK / A_KRow / K1; - // KWmma->MRepeat->MWave->K0PerWmma->KRow->MPerWmma->K1 Per Thread - return make_naive_tensor_descriptor( - make_tuple(Number{}, -@@ -217,8 +218,10 @@ struct GridwiseGemm_Wmma - } - else - { -+ -+ constexpr auto B_KRow = I2; - constexpr auto KWmmaPerblock = KPerBlock / WmmaK; -- constexpr auto K0PerWmma = WmmaK / 2 / K1; -+ constexpr auto K0PerWmma = WmmaK / B_KRow / K1; - // KWmma->NRepeat->MWave->K0PerWmma->KRow->MPerWmma->K1 Per Thread - return make_naive_tensor_descriptor( - make_tuple(Number{}, -@@ -290,12 +293,17 @@ struct GridwiseGemm_Wmma - if constexpr(AEnableLds) - { - // AK0_M_AK1 -> AK0_MRepeat_Mwaves_AKRow_MPerWmma_AK1 -- constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -- constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+ constexpr auto A_K0 = ABlockDesc_{}.GetLength(I0); -+ constexpr auto A_K1 = ABlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto A_KRow = I2; -+#else - constexpr auto A_KRow = I1; -+#endif -+ - return transform_tensor_descriptor( - ABlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, A_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -348,12 +356,16 @@ struct GridwiseGemm_Wmma - if constexpr(BEnableLds) - { - // BK0_N_BK1 -> BK0_NRepeat_Nwaves_NPerWmma_BK1 -- constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -- constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+ constexpr auto B_K0 = BBlockDesc_{}.GetLength(I0); -+ constexpr auto B_K1 = BBlockDesc_{}.GetLength(I2); -+#ifdef __gfx12__ -+ constexpr auto B_KRow = I2; -+#else - constexpr auto B_KRow = I1; -+#endif - return transform_tensor_descriptor( - BBlockDesc_{}, -- make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), -+ make_tuple(make_unmerge_transform(make_tuple(Number{}, B_KRow)), - make_unmerge_transform(make_tuple( - Number{}, Number{}, Number{})), - make_pass_through_transform(Number{})), -@@ -522,12 +534,6 @@ struct GridwiseGemm_Wmma - c_grid_desc_m_n); - } - -- using CGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock = -- remove_cvref_t; -- using DefaultBlock2CTileMap = -- remove_cvref_t; -- - struct SharedMemTrait - { - // LDS allocation for A and B: be careful of alignment -@@ -559,6 +565,12 @@ struct GridwiseGemm_Wmma - b_block_space_size_aligned * sizeof(BDataType)); - }; - -+ using CGridDescriptor_MBlock_MPerBlock_NBlock_NPerBlock = -+ remove_cvref_t; -+ using DefaultBlock2CTileMap = -+ remove_cvref_t; -+ - template - __device__ static void Run(const ADataType* __restrict__ p_a_grid, - const BDataType* __restrict__ p_b_grid, -diff --git a/include/ck/tensor_operation/gpu/grid/gridwise_tensor_rearrange.hpp b/include/ck/tensor_operation/gpu/grid/gridwise_tensor_rearrange.hpp -index 6772524e0..174074990 100644 ---- a/include/ck/tensor_operation/gpu/grid/gridwise_tensor_rearrange.hpp -+++ b/include/ck/tensor_operation/gpu/grid/gridwise_tensor_rearrange.hpp -@@ -35,8 +35,9 @@ __global__ void - const Block2ETileMap block_2_tile_map, - const ComputePtrOffsetOfStridedBatch compute_ptr_offset_of_batch) - { --#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -- defined(__gfx90a__) || defined(__gfx94__) || defined(__gfx103__) || defined(__gfx11__)) -+#if(!defined(__HIP_DEVICE_COMPILE__) || defined(__gfx906__) || defined(__gfx908__) || \ -+ defined(__gfx90a__) || defined(__gfx94__) || defined(__gfx103__) || defined(__gfx11__) || \ -+ defined(__gfx12__)) - GridwiseTensorRearrangeKernel::Run(in_grid_desc, - p_in_global, - out_grid_desc, -diff --git a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp -index bcce930fc..d7a6a3624 100644 ---- a/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp -+++ b/include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer.hpp -@@ -1304,7 +1304,7 @@ struct ThreadwiseTensorSliceTransfer_StaticToStatic - ElementwiseOperation element_op_; - }; - --// Specilized for WMMA -+// Specilized for WMMA-Navi3 - // A single Wave32 is composed by double row - // Data exchange allowed between these two rows - // This RowLane Dst buf will be filled from two Src buf -@@ -1439,4 +1439,111 @@ struct ThreadwiseTensorSliceTransfer_StaticToStatic_InterRow - ElementwiseOperation element_op_{}; - }; - -+// Specilized for WMMA-Navi4 -+template ::type = false> -+struct ThreadwiseTensorSliceTransfer_StaticToStatic_IntraRow -+{ -+ static constexpr index_t nDim = SliceLengths::Size(); -+ -+ using Index = MultiIndex; -+ -+ __device__ constexpr ThreadwiseTensorSliceTransfer_StaticToStatic_IntraRow(const Index& src_idx) -+ { -+ static_assert(SrcDesc::IsKnownAtCompileTime() && DstDesc::IsKnownAtCompileTime(), -+ "wrong! Desc need to known at compile-time"); -+ -+ static_assert(SliceLengths::At(Number{}) % DstScalarPerVector == 0, -+ "wrong! Not divisible"); -+ ignore = src_idx; -+ } -+ -+ template -+ __device__ void Run(const SrcDesc&, -+ const SrcSliceOriginIdx&, -+ const SrcBuffer& src_buf, -+ const DstDesc&, -+ const DstSliceOriginIdx&, -+ DstBuffer& dst_buf) const -+ { -+ static_assert(SrcDesc::IsKnownAtCompileTime() && DstDesc::IsKnownAtCompileTime(), -+ "wrong! Desc need to known at compile-time"); -+ -+ static_assert(is_known_at_compile_time>::value && -+ is_known_at_compile_time>::value, -+ "wrong! SliceOrigin need to known at compile-time"); -+ -+ static_assert(SrcBuffer::IsStaticBuffer() && DstBuffer::IsStaticBuffer(), -+ "wrong! Buffer need to be StaticBuffer"); -+ -+ // SrcDesc and src_slice_origin_idx are known at compile-time -+ constexpr auto src_desc = remove_cvref_t{}; -+ constexpr auto dst_desc = remove_cvref_t{}; -+ constexpr auto src_slice_origin_idx = to_multi_index(SrcSliceOriginIdx{}); -+ constexpr auto dst_slice_origin_idx = to_multi_index(DstSliceOriginIdx{}); -+ -+ // scalar per access on each dim -+ constexpr auto dst_scalar_per_access = generate_sequence( -+ detail::lambda_scalar_per_access{}, Number{}); -+ -+ constexpr auto dst_scalar_step_in_vector = -+ generate_sequence(detail::lambda_scalar_step_in_vector{}, Number{}); -+ -+ using SpaceFillingCurve = SpaceFillingCurve>; -+ -+ static_assert(DstScalarPerVector == SpaceFillingCurve::ScalarPerVector, -+ "wrong!DstScalarPerVector != SpaceFillingCurve::ScalarPerVector"); -+ -+ constexpr auto num_access = SpaceFillingCurve::GetNumOfAccess(); -+ -+ static_for<0, num_access, 1>{}([&](auto idx_1d) { -+ constexpr auto idx_md = SpaceFillingCurve::GetIndex(idx_1d); -+ -+ // copy data from src_buf into dst_vector -+ static_for<0, DstScalarPerVector, 1>{}([&](auto i) { -+ // src_desc error, non constexpr, caused by merge transform -+ constexpr index_t src_offset = src_desc.CalculateOffset( -+ src_slice_origin_idx + idx_md + i * dst_scalar_step_in_vector); -+ -+ constexpr index_t dst_offset = dst_desc.CalculateOffset( -+ dst_slice_origin_idx + idx_md + i * dst_scalar_step_in_vector); -+ -+ SrcData v_this_row; -+ // int type temp value due to intrinsic requirement -+ int temp = 0; -+ -+ // apply element-wise operation -+ element_op_(v_this_row, src_buf[Number{}]); -+ -+ // apply intra-row permute. -+ if constexpr(IntraRowSwizzlePerm) -+ { -+ temp = __builtin_amdgcn_permlane16( -+ temp, type_convert_sp(v_this_row), 0xb3a29180, 0xf7e6d5c4, 1, 0); -+ v_this_row = type_convert_sp(temp); -+ } -+ -+ // apply type convert -+ dst_buf(Number{}) = type_convert_sp(v_this_row); -+ }); -+ }); -+ } -+ ElementwiseOperation element_op_{}; -+}; -+ - } // namespace ck -diff --git a/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp b/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp -index 565195f53..9a9ebf559 100644 ---- a/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp -+++ b/include/ck/tensor_operation/gpu/warp/wmma_gemm.hpp -@@ -11,12 +11,17 @@ namespace ck { - - enum struct WmmaInstr - { -+ // gfx11 - wmma_f32_16x16x16_f16 = 0, - wmma_f32_16x16x16_bf16, - wmma_f16_16x16x16_f16, - wmma_bf16_16x16x16_bf16, - wmma_i32_16x16x16_iu8, -- wmma_i32_16x16x16_iu4 -+ wmma_i32_16x16x16_iu4, -+ // gfx12 -+ wmma_f32_16x16x16_f16_gfx12, -+ wmma_f32_16x16x16_bf16_gfx12, -+ wmma_i32_16x16x16_iu8_gfx12, - }; - - /* -@@ -279,6 +284,122 @@ struct wmma_type -+struct wmma_type> -+{ -+ // Absolute fixing property -+ // * Data Pixel -+ static constexpr index_t m_per_wmma = 16; -+ static constexpr index_t n_per_wmma = 16; -+ static constexpr index_t k_per_wmma = 16; -+ // static constexpr index_t src_a_data_size = 2; -+ // static constexpr index_t src_b_data_size = 2; -+ // static constexpr index_t acc_data_size = 4; -+ // * Thread mapping inside wave, num_thread_per_subgroups always alone N direction -+ static constexpr index_t acc_data_size = 4; -+ static constexpr index_t acc_pack_number = 1; -+ static constexpr index_t num_thread_per_subgroups = n_per_wmma; -+ -+ // Wave mode dependent propety -+ static constexpr index_t wave_size = Number{}; -+ // * Fixed in Navi3x, Will be wave mode dependent on Navi4x -+ // static constexpr index_t num_src_a_vgprs_per_wave = k_per_wmma / 2 * src_a_data_size / 4; -+ // static constexpr index_t num_src_b_vgprs_per_wave = k_per_wmma / 2 * src_b_data_size / 4; -+ // * num_acc_vgprs_per_wave alone M direction -+ // * num_subgroups alone M direction -+ static constexpr index_t num_acc_vgprs_per_wave = m_per_wmma * n_per_wmma / wave_size; -+ static constexpr index_t num_subgroups = wave_size / num_thread_per_subgroups; -+ -+ template -+ __device__ void run(const FloatA& a, const FloatB& b, FloatC& reg_c) const -+ { -+ static_assert(wave_size == 32, "only support wave32 for gfx12 wmma"); -+ if constexpr(wave_size == 32) -+ { -+ intrin_wmma_f32_16x16x16_f16_w32_gfx12::Run(a, b, reg_c); -+ } -+ } -+}; -+ -+template -+struct wmma_type> -+{ -+ // Absolute fixing property -+ static constexpr index_t m_per_wmma = 16; -+ static constexpr index_t n_per_wmma = 16; -+ static constexpr index_t k_per_wmma = 16; -+ // static constexpr index_t src_a_data_size = 2; -+ // static constexpr index_t src_b_data_size = 2; -+ static constexpr index_t acc_data_size = 4; -+ static constexpr index_t acc_pack_number = 1; -+ static constexpr index_t num_thread_per_subgroups = n_per_wmma; -+ -+ // Wave mode dependent propety -+ static constexpr index_t wave_size = Number{}; -+ // static constexpr index_t num_src_a_vgprs_per_wave = m_per_wmma * src_a_data_size / 4; -+ // static constexpr index_t num_src_b_vgprs_per_wave = n_per_wmma * src_b_data_size / 4; -+ static constexpr index_t num_acc_vgprs_per_wave = m_per_wmma * n_per_wmma / wave_size; -+ static constexpr index_t num_subgroups = wave_size / num_thread_per_subgroups; -+ -+ template -+ __device__ void run(const FloatA& a, const FloatB& b, FloatC& reg_c) const -+ { -+ static_assert(wave_size == 32, "only support wave32 for gfx12 wmma"); -+ if constexpr(wave_size == 32) -+ { -+ intrin_wmma_f32_16x16x16_bf16_w32_gfx12::Run(a, b, reg_c); -+ } -+ } -+}; -+ -+template -+struct wmma_type> -+{ -+ // Absolute fixing property -+ static constexpr index_t m_per_wmma = 16; -+ static constexpr index_t n_per_wmma = 16; -+ static constexpr index_t k_per_wmma = 16; -+ // static constexpr index_t src_a_data_size = 2; -+ // static constexpr index_t src_b_data_size = 2; -+ static constexpr index_t acc_data_size = 4; -+ static constexpr index_t acc_pack_number = 1; -+ static constexpr index_t num_thread_per_subgroups = n_per_wmma; -+ -+ // Wave mode dependent propety -+ static constexpr index_t wave_size = Number{}; -+ // static constexpr index_t num_src_a_vgprs_per_wave = m_per_wmma * src_a_data_size / 4; -+ // static constexpr index_t num_src_b_vgprs_per_wave = n_per_wmma * src_b_data_size / 4; -+ static constexpr index_t num_acc_vgprs_per_wave = m_per_wmma * n_per_wmma / wave_size; -+ static constexpr index_t num_subgroups = wave_size / num_thread_per_subgroups; -+ -+ template -+ __device__ void run(const FloatA& a, const FloatB& b, FloatC& reg_c) const -+ { -+ static_assert(wave_size == 32, "only support wave32 for gfx12 wmma"); -+ if constexpr(wave_size == 32) -+ { -+ intrin_wmma_i32_16x16x16_iu8_w32_gfx12::Run( -+ a, b, reg_c); -+ } -+ } -+}; -+ - template - static constexpr auto GetWmma() - { -+#ifdef __gfx12__ -+ return WmmaInstr::wmma_f32_16x16x16_f16_gfx12; -+#else - return WmmaInstr::wmma_f32_16x16x16_f16; -+#endif - } - - template <> - static constexpr auto GetWmma() - { -+#ifdef __gfx12__ -+ return WmmaInstr::wmma_f32_16x16x16_bf16_gfx12; -+#else - return WmmaInstr::wmma_f32_16x16x16_bf16; -+#endif - } - - template <> -@@ -320,8 +449,13 @@ struct WmmaSelector - template <> - static constexpr auto GetWmma() - { -+#ifdef __gfx12__ -+ return WmmaInstr::wmma_i32_16x16x16_iu8_gfx12; -+#else - return WmmaInstr::wmma_i32_16x16x16_iu8; -+#endif - } -+ - #ifdef CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4 - template <> - static constexpr auto GetWmma() -@@ -502,6 +636,9 @@ struct WmmaGemm - - __device__ static auto GetSubGroupId() - { -+ static_assert(wmma_instr.num_thread_per_subgroups * wmma_instr.num_subgroups == -+ wmma_instr.wave_size, -+ ""); - return (GetLaneId() / wmma_instr.num_thread_per_subgroups) % wmma_instr.num_subgroups; - } - -@@ -516,12 +653,20 @@ struct WmmaGemm - - __host__ __device__ static auto CalculateAThreadOriginDataIndex() - { -+#ifdef __gfx12__ -+ return GetLaneIdUnderSubGroup(); -+#else - return TransposeC ? GetLaneIdUnderSubGroup() : GetSwizzledLaneIdLow(); -+#endif - } - - __host__ __device__ static auto CalculateBThreadOriginDataIndex() - { -+#ifdef __gfx12__ -+ return GetLaneIdUnderSubGroup(); -+#else - return TransposeC ? GetSwizzledLaneIdLow() : GetLaneIdUnderSubGroup(); -+#endif - } - - __device__ static CIndex GetBeginOfThreadBlk() -diff --git a/include/ck/utility/amd_wmma.hpp b/include/ck/utility/amd_wmma.hpp -index 1bb0140f3..322a0f94b 100644 ---- a/include/ck/utility/amd_wmma.hpp -+++ b/include/ck/utility/amd_wmma.hpp -@@ -257,5 +257,87 @@ struct intrin_wmma_i32_16x16x16_iu8_w64<16, 16, neg_a, neg_b, clamp> - } - }; - -+// gfx12 -+/********************************WAVE32 MODE***********************************************/ -+ -+#if defined(__gfx1200__) || defined(__gfx1201__) -+#define __gfx12__ -+#endif -+ -+// src: fp16, dst: fp32 -+template -+struct intrin_wmma_f32_16x16x16_f16_w32_gfx12; -+ -+template <> -+struct intrin_wmma_f32_16x16x16_f16_w32_gfx12<16, 16> -+{ -+ template -+ __device__ static void Run(const half8_t& reg_a, const half8_t& reg_b, FloatC& reg_c) -+ { -+ // * Inline assembly need to elimate the duplicated data load, compiler won't help you -+ // delete them. -+ // amd_assembly_wmma_f32_16x16x16_f16_w32( -+ // reg_a, reg_b, reg_c.template AsType()(Number<0>{})); -+#if defined(__gfx12__) -+ reg_c.template AsType()(Number<0>{}) = -+ __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12( -+ reg_a, reg_b, reg_c.template AsType()[Number<0>{}]); -+#else -+ ignore = reg_a; -+ ignore = reg_b; -+ ignore = reg_c; -+#endif -+ } -+}; -+ -+// src: bf16, dst: fp32 -+template -+struct intrin_wmma_f32_16x16x16_bf16_w32_gfx12; -+ -+template <> -+struct intrin_wmma_f32_16x16x16_bf16_w32_gfx12<16, 16> -+{ -+ template -+ __device__ static void Run(const bhalf8_t& reg_a, const bhalf8_t& reg_b, FloatC& reg_c) -+ { -+#if defined(__gfx12__) -+ reg_c.template AsType()(Number<0>{}) = -+ __builtin_amdgcn_wmma_f32_16x16x16_bf16_w32_gfx12( -+ reg_a, reg_b, reg_c.template AsType()[Number<0>{}]); -+#else -+ ignore = reg_a; -+ ignore = reg_b; -+ ignore = reg_c; -+#endif -+ } -+}; -+ -+// src: iu8, dst: i32 -+template -+struct intrin_wmma_i32_16x16x16_iu8_w32_gfx12; -+ -+template -+struct intrin_wmma_i32_16x16x16_iu8_w32_gfx12<16, 16, neg_a, neg_b, clamp> -+{ -+ template -+ __device__ static void Run(const int8x8_t& reg_a, const int8x8_t& reg_b, FloatC& reg_c) -+ { -+#if defined(__gfx12__) -+ reg_c.template AsType()(Number<0>{}) = -+ __builtin_amdgcn_wmma_i32_16x16x16_iu8_w32_gfx12( -+ neg_a, -+ bit_cast(reg_a), -+ neg_b, -+ bit_cast(reg_b), -+ reg_c.template AsType()[Number<0>{}], -+ clamp); -+#else -+ ignore = reg_a; -+ ignore = reg_b; -+ ignore = reg_c; -+#endif -+ } -+}; -+ - } // namespace ck - #endif -diff --git a/include/ck/utility/data_type.hpp b/include/ck/utility/data_type.hpp -index 93a1edefb..4df14c621 100644 ---- a/include/ck/utility/data_type.hpp -+++ b/include/ck/utility/data_type.hpp -@@ -203,7 +203,7 @@ struct vector_type - } - }; - --int static err = 0; -+__device__ int static err = 0; - template - struct vector_type - { -diff --git a/include/ck/utility/synchronization.hpp b/include/ck/utility/synchronization.hpp -index 4fe5e3950..d6b6eac26 100644 ---- a/include/ck/utility/synchronization.hpp -+++ b/include/ck/utility/synchronization.hpp -@@ -10,12 +10,20 @@ namespace ck { - __device__ void block_sync_lds() - { - #if CK_EXPERIMENTAL_BLOCK_SYNC_LDS_WITHOUT_SYNC_VMEM -+#ifdef __gfx12__ -+ asm volatile("\ -+ s_wait_dscnt 0x0 \n \ -+ s_barrier_signal -1 \n \ -+ s_barrier_wait -1 \ -+ " ::); -+#else - // asm volatile("\ - // s_waitcnt lgkmcnt(0) \n \ - // s_barrier \ - // " ::); - __builtin_amdgcn_s_waitcnt(0xc07f); - __builtin_amdgcn_s_barrier(); -+#endif - #else - __syncthreads(); - #endif -@@ -23,11 +31,20 @@ __device__ void block_sync_lds() - - __device__ void block_sync_lds_direct_load() - { -+#ifdef __gfx12__ -+ asm volatile("\ -+ s_wait_vmcnt 0x0 \n \ -+ s_wait_dscnt 0x0 \n \ -+ s_barrier_signal -1 \n \ -+ s_barrier_wait -1 \ -+ " ::); -+#else - asm volatile("\ - s_waitcnt vmcnt(0) \n \ - s_waitcnt lgkmcnt(0) \n \ - s_barrier \ - " ::); -+#endif - } - - __device__ void s_nop() -diff --git a/include/ck_tile/core/config.hpp b/include/ck_tile/core/config.hpp -index 601aad19b..9dc2b072a 100644 ---- a/include/ck_tile/core/config.hpp -+++ b/include/ck_tile/core/config.hpp -@@ -17,6 +17,9 @@ - #if defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1102__) || defined(__gfx1103__) - #define __gfx11__ - #endif -+#if defined(__gfx1200__) || defined(__gfx1201__) -+#define __gfx12__ -+#endif - - #ifndef CK_TILE_DONT_USE_HIP_RUNTIME_HEADERS - #include "hip/hip_runtime.h" -@@ -155,7 +158,7 @@ - #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x00020000 - #elif defined(__gfx103__) // for GPU code - #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x31014000 --#elif defined(__gfx11__) // for GPU code -+#elif defined(__gfx11__) || defined(__gfx12__) // for GPU code - #define CK_TILE_BUFFER_RESOURCE_3RD_DWORD 0x31004000 - #endif - -diff --git a/library/src/tensor_operation_instance/gpu/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/CMakeLists.txt -index 8c5f36d2e..89c9d6dc6 100644 ---- a/library/src/tensor_operation_instance/gpu/CMakeLists.txt -+++ b/library/src/tensor_operation_instance/gpu/CMakeLists.txt -@@ -52,7 +52,7 @@ function(add_instance_library INSTANCE_NAME) - endforeach() - # Do not build WMMA instances if gfx11 targets are not on the target list - foreach(source IN LISTS ARGN) -- if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "_wmma") -+ if(NOT GPU_TARGETS MATCHES "gfx11" AND NOT GPU_TARGETS MATCHES "gfx12" AND source MATCHES "_wmma") - message("removing wmma instance ${source} ") - list(REMOVE_ITEM ARGN "${source}") - endif() -@@ -149,7 +149,7 @@ FOREACH(subdir_path ${dir_list}) - message("Found only xdl instances, but gfx9 is not on the targets list. Skipping.") - set(add_inst 0) - endif() -- if(("${cmake_instance}" MATCHES "ONLY WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11")) -+ if(("${cmake_instance}" MATCHES "ONLY WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11") AND (NOT GPU_TARGETS MATCHES "gfx12")) - message("Found only wmma instances, but gfx11 is not on the targets list. Skipping.") - set(add_inst 0) - endif() -@@ -157,11 +157,11 @@ FOREACH(subdir_path ${dir_list}) - message("Found only xdl and dl instances, but gfx9 is not on the targets listand DL_KERNELS is not set. Skipping.") - set(add_inst 0) - endif() -- if(("${cmake_instance}" MATCHES "ONLY XDL_AND_WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11") AND (NOT GPU_TARGETS MATCHES "gfx9")) -+ if(("${cmake_instance}" MATCHES "ONLY XDL_AND_WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11") AND (NOT GPU_TARGETS MATCHES "gfx12") AND (NOT GPU_TARGETS MATCHES "gfx9")) - message("Found only xdl and wmma instances, but gfx11 and gfx9 are not on the targets list. Skipping.") - set(add_inst 0) - endif() -- if(("${cmake_instance}" MATCHES "XDL_DL_WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11") AND (NOT GPU_TARGETS MATCHES "gfx9") AND (NOT DEFINED DL_KERNELS)) -+ if(("${cmake_instance}" MATCHES "XDL_DL_WMMA_KERNELS") AND (NOT GPU_TARGETS MATCHES "gfx11") AND (NOT GPU_TARGETS MATCHES "gfx12") AND (NOT GPU_TARGETS MATCHES "gfx9") AND (NOT DEFINED DL_KERNELS)) - message("Found xdl, dl, and wmma instances, but none of those meet the target list. Skipping.") - set(add_inst 0) - endif() -diff --git a/profiler/src/CMakeLists.txt b/profiler/src/CMakeLists.txt -index 1cfcbfff6..a9557a9b9 100644 ---- a/profiler/src/CMakeLists.txt -+++ b/profiler/src/CMakeLists.txt -@@ -58,7 +58,7 @@ if(GPU_TARGETS MATCHES "gfx9") - - endif() - --if(GPU_TARGETS MATCHES "gfx11" OR GPU_TARGETS MATCHES "gfx9") -+if(GPU_TARGETS MATCHES "gfx11" OR GPU_TARGETS MATCHES "gfx12" OR GPU_TARGETS MATCHES "gfx9") - if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES) - list(APPEND PROFILER_SOURCES profile_gemm_bilinear.cpp) - endif() -@@ -133,7 +133,7 @@ if(GPU_TARGETS MATCHES "gfx9") - target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_grouped_conv2d_bwd_weight_instance) - endif() - --if(GPU_TARGETS MATCHES "gfx9" OR GPU_TARGETS MATCHES "gfx11") -+if(GPU_TARGETS MATCHES "gfx9" OR GPU_TARGETS MATCHES "gfx11" OR GPU_TARGETS MATCHES "gfx12") - if(DTYPES MATCHES "fp16" OR NOT DEFINED DTYPES) - target_link_libraries(${PROFILER_EXECUTABLE} PRIVATE device_gemm_bilinear_instance) - endif() -diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt -index 25c63ac7f..2a7c52b58 100644 ---- a/test/CMakeLists.txt -+++ b/test/CMakeLists.txt -@@ -53,7 +53,7 @@ function(add_test_executable TEST_NAME) - endif() - endforeach() - foreach(source IN LISTS ARGN) -- if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "wmma") -+ if(NOT GPU_TARGETS MATCHES "gfx11" AND NOT GPU_TARGETS MATCHES "gfx12" AND source MATCHES "wmma") - message("removing wmma test ${source} ") - list(REMOVE_ITEM ARGN "${source}") - endif() -@@ -118,7 +118,7 @@ function(add_gtest_executable TEST_NAME) - endif() - endforeach() - foreach(source IN LISTS ARGN) -- if(NOT GPU_TARGETS MATCHES "gfx11" AND source MATCHES "wmma") -+ if(NOT GPU_TARGETS MATCHES "gfx11" AND NOT GPU_TARGETS MATCHES "gfx12" AND source MATCHES "wmma") - message("removing wmma test ${source} ") - list(REMOVE_ITEM ARGN "${source}") - endif() -diff --git a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp -index 1c8082645..21f49ec0f 100644 ---- a/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp -+++ b/test/grouped_convnd_bwd_weight/test_grouped_convnd_bwd_weight.cpp -@@ -55,7 +55,7 @@ class TestGroupedConvndBwdWeight : public ::testing::Test - } - } - -- if(ck::is_gfx11_supported()) -+ if(ck::is_gfx11_supported() || ck::is_gfx12_supported()) - { - // on gfx11 only support for 3d is implemented - if constexpr(NDimSpatial{} != 3) -diff --git a/test/wmma_op/wmma_op_util.hpp b/test/wmma_op/wmma_op_util.hpp -index 49782bce6..d9ec94771 100644 ---- a/test/wmma_op/wmma_op_util.hpp -+++ b/test/wmma_op/wmma_op_util.hpp -@@ -140,10 +140,18 @@ __global__ void matmul(const src_t* a, const src_t* b, dst_t* c) - p_shared[8 * 16 * lane_hi + 8 * lane_lo + ele + 16 * 16] = b_temp[ele]; - } - -+#ifdef __gfx12__ -+ asm volatile("\ -+ s_wait_dscnt 0x0 \n \ -+ s_barrier_signal -1 \n \ -+ s_barrier_wait -1 \ -+ " ::); -+#else - asm volatile("\ - s_waitcnt lgkmcnt(0) \n \ - s_barrier \ - " ::); -+#endif - - for(int ele = 0; ele < 16; ++ele) - { -@@ -155,10 +163,18 @@ __global__ void matmul(const src_t* a, const src_t* b, dst_t* c) - a_frag[ele] = p_shared[(ele / 8) * 16 * 8 + 8 * lane + ele % 8]; - } - -+#ifdef __gfx12__ -+ asm volatile("\ -+ s_wait_dscnt 0x0 \n \ -+ s_barrier_signal -1 \n \ -+ s_barrier_wait -1 \ -+ " ::); -+#else - asm volatile("\ - s_waitcnt lgkmcnt(0) \n \ - s_barrier \ - " ::); -+#endif - - // sync threads, similar to mma_sync - // __syncthreads(); diff --git a/cmake/patches/composable_kernel/Fix_Clang_Build.patch b/cmake/patches/composable_kernel/Fix_Clang_Build.patch deleted file mode 100644 index d63da63445fde..0000000000000 --- a/cmake/patches/composable_kernel/Fix_Clang_Build.patch +++ /dev/null @@ -1,238 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index c23746e7f..bc326c8b5 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -23,10 +23,10 @@ endif() - - set(version 1.1.0) - # Check support for CUDA/HIP in Cmake --project(composable_kernel VERSION ${version} LANGUAGES CXX) -+project(composable_kernel VERSION ${version} LANGUAGES CXX HIP) - include(CTest) - --find_package(Python3 3.6 COMPONENTS Interpreter REQUIRED) -+find_package(Python3 COMPONENTS Interpreter REQUIRED) - - list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") - -@@ -227,27 +227,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(CMAKE_CXX_EXTENSIONS OFF) - message("CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") - --## OpenMP --if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") -- # workaround issue hipcc in rocm3.5 cannot find openmp -- set(OpenMP_CXX "${CMAKE_CXX_COMPILER}") -- set(OpenMP_CXX_FLAGS "-fopenmp=libomp -Wno-unused-command-line-argument") -- set(OpenMP_CXX_LIB_NAMES "libomp" "libgomp" "libiomp5") -- set(OpenMP_libomp_LIBRARY ${OpenMP_CXX_LIB_NAMES}) -- set(OpenMP_libgomp_LIBRARY ${OpenMP_CXX_LIB_NAMES}) -- set(OpenMP_libiomp5_LIBRARY ${OpenMP_CXX_LIB_NAMES}) --else() -- find_package(OpenMP REQUIRED) --endif() -- --message("OpenMP_CXX_LIB_NAMES: ${OpenMP_CXX_LIB_NAMES}") --message("OpenMP_gomp_LIBRARY: ${OpenMP_gomp_LIBRARY}") --message("OpenMP_pthread_LIBRARY: ${OpenMP_pthread_LIBRARY}") --message("OpenMP_CXX_FLAGS: ${OpenMP_CXX_FLAGS}") -- --link_libraries(${OpenMP_gomp_LIBRARY}) --link_libraries(${OpenMP_pthread_LIBRARY}) -- - ## HIP - find_package(HIP REQUIRED) - # Override HIP version in config.h, if necessary. -@@ -269,12 +248,6 @@ if( DEFINED CK_OVERRIDE_HIP_VERSION_PATCH ) - message(STATUS "CK_HIP_VERSION_PATCH overridden with ${CK_OVERRIDE_HIP_VERSION_PATCH}") - endif() - message(STATUS "Build with HIP ${HIP_VERSION}") --link_libraries(hip::device) --if(CK_hip_VERSION VERSION_GREATER_EQUAL 6.0.23494) -- add_compile_definitions(__HIP_PLATFORM_AMD__=1) --else() -- add_compile_definitions(__HIP_PLATFORM_HCC__=1) --endif() - - ## tidy - include(EnableCompilerWarnings) -@@ -541,11 +514,3 @@ rocm_install(FILES - - set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") - set(CPACK_RPM_PACKAGE_LICENSE "MIT") -- --rocm_create_package( -- NAME composablekernel -- DESCRIPTION "High Performance Composable Kernel for AMD GPUs" -- MAINTAINER "MIOpen Kernels Dev Team " -- LDCONFIG -- HEADER_ONLY --) -diff --git a/example/ck_tile/01_fmha/generate.py b/example/ck_tile/01_fmha/generate.py -index 51fecd07b..5ed371995 100644 ---- a/example/ck_tile/01_fmha/generate.py -+++ b/example/ck_tile/01_fmha/generate.py -@@ -566,7 +566,7 @@ def write_blobs(output_dir : Optional[str], kernel_filter : Optional[str], recei - def list_blobs(output_file : Optional[str], kernel_filter : Optional[str], receipt, mask_impl) -> None: - assert output_file is not None - file_path = Path(output_file) -- with file_path.open('a') as f: -+ with file_path.open('w') as f: - _, kernels = get_blobs(kernel_filter, receipt, mask_impl) - for kernel in kernels: - f.write(str(file_path.parent / GEN_DIR / kernel.filename) + "\n") -diff --git a/include/ck/host_utility/hip_check_error.hpp b/include/ck/host_utility/hip_check_error.hpp -index c0894f1d7..559481fee 100644 ---- a/include/ck/host_utility/hip_check_error.hpp -+++ b/include/ck/host_utility/hip_check_error.hpp -@@ -6,19 +6,7 @@ - #include - #include - --// To be removed, which really does not tell the location of failed HIP functional call --inline void hip_check_error(hipError_t x) --{ -- if(x != hipSuccess) -- { -- std::ostringstream ss; -- ss << "HIP runtime error: " << hipGetErrorString(x) << ". " -- << "hip_check_error.hpp" -- << ": " << __LINE__ << "in function: " << __func__; -- throw std::runtime_error(ss.str()); -- } --} -- -+#ifndef HIP_CHECK_ERROR - #define HIP_CHECK_ERROR(retval_or_funcall) \ - do \ - { \ -@@ -32,3 +20,9 @@ inline void hip_check_error(hipError_t x) - throw std::runtime_error(ostr.str()); \ - } \ - } while(0) -+#endif -+ -+#ifndef hip_check_error -+#define hip_check_error HIP_CHECK_ERROR -+#endif -+ -diff --git a/include/ck_tile/core/utility/transpose_vectors.hpp b/include/ck_tile/core/utility/transpose_vectors.hpp -index a164c3f94..293ead89a 100644 ---- a/include/ck_tile/core/utility/transpose_vectors.hpp -+++ b/include/ck_tile/core/utility/transpose_vectors.hpp -@@ -11,6 +11,9 @@ - - namespace ck_tile { - -+template -+constexpr bool always_false = false; -+ - // S: scalar type (or it can be non-scalar type) - // NX: # of vector before transpose - // NY: # of vector after transpose -@@ -117,9 +120,11 @@ struct transpose_vectors - } - else - { -- static_assert(false, "not implemented"); -+ static_assert(always_false, number>, "not implemented"); - } - } - }; - -+ - } // namespace ck_tile -+ -diff --git a/include/ck_tile/host/hip_check_error.hpp b/include/ck_tile/host/hip_check_error.hpp -index 3acdb4d87..cc26e184f 100644 ---- a/include/ck_tile/host/hip_check_error.hpp -+++ b/include/ck_tile/host/hip_check_error.hpp -@@ -8,20 +8,7 @@ - #include - #include - --namespace ck_tile { --// To be removed, which really does not tell the location of failed HIP functional call --CK_TILE_HOST void hip_check_error(hipError_t x) --{ -- if(x != hipSuccess) -- { -- std::ostringstream ss; -- ss << "HIP runtime error: " << hipGetErrorString(x) << ". " << __FILE__ << ": " << __LINE__ -- << "in function: " << __func__; -- throw std::runtime_error(ss.str()); -- } --} --} // namespace ck_tile -- -+#ifndef HIP_CHECK_ERROR - #define HIP_CHECK_ERROR(retval_or_funcall) \ - do \ - { \ -@@ -34,3 +21,9 @@ CK_TILE_HOST void hip_check_error(hipError_t x) - throw std::runtime_error(ostr.str()); \ - } \ - } while(0) -+#endif -+ -+#ifndef hip_check_error -+#define hip_check_error HIP_CHECK_ERROR -+#endif -+ -diff --git a/library/src/tensor_operation_instance/gpu/CMakeLists.txt b/library/src/tensor_operation_instance/gpu/CMakeLists.txt -index c035e7e56..8c5f36d2e 100644 ---- a/library/src/tensor_operation_instance/gpu/CMakeLists.txt -+++ b/library/src/tensor_operation_instance/gpu/CMakeLists.txt -@@ -59,8 +59,14 @@ function(add_instance_library INSTANCE_NAME) - endforeach() - #only continue if there are some source files left on the list - if(ARGN) -+ set_source_files_properties(${ARGN} PROPERTIES LANGUAGE HIP) - add_library(${INSTANCE_NAME} OBJECT ${ARGN}) -+ # Always disable debug symbol and C debug assert due to -+ # - Linker error: ... relocation truncated to fit ..., caused by object files to be linked are too huge. -+ # - https://github.com/ROCmSoftwarePlatform/composable_kernel/issues/622 -+ target_compile_options(${INSTANCE_NAME} PRIVATE -g0 -DNDEBUG) - target_compile_features(${INSTANCE_NAME} PUBLIC) -+ target_compile_definitions(${INSTANCE_NAME} PRIVATE "__HIP_PLATFORM_AMD__=1" "__HIP_PLATFORM_HCC__=1") - set_target_properties(${INSTANCE_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) - clang_tidy_check(${INSTANCE_NAME}) - set(result 0) ---- ./include/ck/utility/amd_buffer_addressing.hpp 2024-09-05 10:12:33.343091000 +0800 -+++ ./include/ck/utility/amd_buffer_addressing_new.hpp 2024-09-05 10:12:20.276686000 +0800 -@@ -991,7 +991,8 @@ - asm volatile("s_mov_b32 m0, %0; \n\t" - "buffer_load_dword %1, %2, 0 offen lds;\n\t" ::"s"(lds_ptr_sgpr), - "v"(global_offset_bytes), -- "s"(src_resource)); -+ "s"(src_resource) -+ : "memory"); - #else - // LDS pointer must be attributed with the LDS address space. - __attribute__((address_space(3))) uint32_t* lds_ptr = ---- ./include/ck_tile/core/arch/amd_buffer_addressing.hpp 2024-09-05 10:18:28.884031000 +0800 -+++ ./include/ck_tile/core/arch/amd_buffer_addressing_new.hpp 2024-09-05 10:17:29.434931000 +0800 -@@ -26,7 +26,12 @@ - CK_TILE_DEVICE int32x4_t make_wave_buffer_resource(const void* ptr, uint32_t size = 0xffffffff) - { - buffer_resource res{ptr, size, CK_TILE_BUFFER_RESOURCE_3RD_DWORD}; -- return __builtin_bit_cast(int32x4_t, res); -+ int32x4_t r = __builtin_bit_cast(int32x4_t, res); -+ r.x = __builtin_amdgcn_readfirstlane(r.x); -+ r.y = __builtin_amdgcn_readfirstlane(r.y); -+ r.z = __builtin_amdgcn_readfirstlane(r.z); -+ r.w = __builtin_amdgcn_readfirstlane(r.w); -+ return r; - } - - // TODO: glc/slc/... -@@ -2016,7 +2021,8 @@ - asm volatile("s_mov_b32 m0, %0; \n\t" - "buffer_load_dword %1, %2, 0 offen lds;\n\t" ::"s"(lds_ptr_sgpr), - "v"(global_offset_bytes), -- "s"(src_resource)); -+ "s"(src_resource) -+ : "memory"); - #else - // LDS pointer must be attributed with the LDS address space. - __attribute__((address_space(3))) uint32_t* lds_ptr = diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs index 73613541f8362..3779a72d4de69 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs @@ -164,10 +164,6 @@ public void TestSessionOptions() opt.AppendExecutionProvider_OpenVINO(); #endif -#if USE_ROCM - opt.AppendExecutionProvider_ROCm(0); -#endif - #if USE_TENSORRT opt.AppendExecutionProvider_Tensorrt(0); #endif @@ -1764,33 +1760,6 @@ void TestCUDAAllocatorInternal(InferenceSession session) } #endif -#if USE_ROCM - void TestROCMAllocatorInternal(InferenceSession session) - { - int device_id = 0; - using (var info_rocm = new OrtMemoryInfo(OrtMemoryInfo.allocatorHIP, OrtAllocatorType.ArenaAllocator, device_id, OrtMemType.Default)) - { - Assert.Equal("Hip", info_rocm.Name); - Assert.Equal(device_id, info_rocm.Id); - Assert.Equal(OrtAllocatorType.ArenaAllocator, info_rocm.GetAllocatorType()); - Assert.Equal(OrtMemType.Default, info_rocm.GetMemoryType()); - - using (var allocator = new OrtAllocator(session, info_rocm)) - { - var alloc_info = allocator.Info; - Assert.True(info_rocm.Equals(alloc_info)); - - uint size = 1024; - OrtMemoryAllocation chunk = allocator.Allocate(size); - Assert.Equal(chunk.Size, size); - Assert.True(chunk.Info.Equals(alloc_info)); - chunk.Dispose(); - alloc_info.Dispose(); - } - } - } -#endif - [Fact(DisplayName = "TestAllocator")] private void TestAllocator() { @@ -1801,21 +1770,12 @@ private void TestAllocator() #if USE_CUDA options.AppendExecutionProvider_CUDA(0); #endif - -#if USE_ROCM - options.AppendExecutionProvider_ROCm(0); -#endif - using (var session = new InferenceSession(model, options)) { TestCPUAllocatorInternal(session); #if USE_CUDA TestCUDAAllocatorInternal(session); #endif -#if USE_ROCM - TestROCMAllocatorInternal(session); -#endif - } } } @@ -1942,15 +1902,6 @@ internal static Tuple, float[]> Op { option.AppendExecutionProvider_CPU(1); } -#elif USE_ROCM - using (var option = (deviceId.HasValue) ? - SessionOptions.MakeSessionOptionWithRocmProvider(deviceId.Value) : - new SessionOptions()) - { - if(!deviceId.HasValue) - { - option.AppendExecutionProvider_CPU(1); - } #else using (var option = new SessionOptions()) { diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/OrtEnvTests.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/OrtEnvTests.cs index ae4fb0cf164cd..94f8e927c1331 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/OrtEnvTests.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/OrtEnvTests.cs @@ -60,9 +60,6 @@ public void GetAvailableProviders() #if USE_CUDA Assert.True(Array.Exists(providers, provider => provider == "CUDAExecutionProvider")); -#endif -#if USE_ROCM - Assert.True(Array.Exists(providers, provider => provider == "ROCMExecutionProvider")); #endif } } @@ -493,4 +490,3 @@ void TestCopyTensors() } } } - diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index 89dbce05326b5..f0d1313783643 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -1531,7 +1531,6 @@ private void TestInferenceWithLoraAdapterFromArray() // TestGpu() will test // - the CUDA EP on CUDA enabled builds // - the DML EP on DML enabled builds - // - the ROCm EP on ROCm enabled builds [GpuFact(DisplayName = "TestGpu")] private void TestGpu() { @@ -1575,9 +1574,6 @@ private void VerifyNativeMethodsExist() #if USE_CUDA ,"OrtSessionOptionsAppendExecutionProvider_CUDA" #endif -#if USE_ROCM - ,"OrtSessionOptionsAppendExecutionProvider_ROCM" -#endif #if USE_DML ,"OrtSessionOptionsAppendExecutionProvider_DML" #endif diff --git a/dockerfiles/Dockerfile.migraphx b/dockerfiles/Dockerfile.migraphx index 876a07e4ffaf6..3048217601b0f 100644 --- a/dockerfiles/Dockerfile.migraphx +++ b/dockerfiles/Dockerfile.migraphx @@ -22,5 +22,5 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh onnxruntime/dockerfiles/scripts/install_common_deps.sh &&\ cd onnxruntime && pip install --upgrade pip &&\ /bin/sh ./build.sh --allow_running_as_root --cmake_extra_defines ONNXRUNTIME_VERSION=`cat ./VERSION_NUMBER` --config Release --parallel \ - --skip_tests --build_wheel --use_rocm --rocm_version=${ROCM_VERSION} --rocm_home /opt/rocm --use_migraphx &&\ + --skip_tests --build_wheel --use_migraphx &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl diff --git a/include/onnxruntime/core/framework/op_kernel.h b/include/onnxruntime/core/framework/op_kernel.h index e59a803d97629..5f391432ce503 100644 --- a/include/onnxruntime/core/framework/op_kernel.h +++ b/include/onnxruntime/core/framework/op_kernel.h @@ -190,13 +190,6 @@ KernelCreateInfo BuildKernelCreateInfo(); } // namespace js } // namespace contrib -namespace contrib { -namespace rocm { -template -KernelCreateInfo BuildKernelCreateInfo(); -} // namespace rocm -} // namespace contrib - namespace contrib { namespace snpe { template diff --git a/include/onnxruntime/core/framework/ortdevice.h b/include/onnxruntime/core/framework/ortdevice.h index 935be9c3f00c7..c85b01210fc3b 100644 --- a/include/onnxruntime/core/framework/ortdevice.h +++ b/include/onnxruntime/core/framework/ortdevice.h @@ -56,7 +56,7 @@ struct OrtDevice { enum VendorIds : VendorId { // No vendor ID. Valid for DeviceType::CPU + MemType::DEFAULT or for generic allocators like WebGPU. NONE = 0x0000, - AMD = 0x1002, // ROCm, MIGraphX EPs + AMD = 0x1002, // MIGraphX EP NVIDIA = 0x10DE, // CUDA/TensorRT ARM = 0x13B5, // ARM GPU EP MICROSOFT = 0x1414, // DML EP diff --git a/include/onnxruntime/core/graph/constants.h b/include/onnxruntime/core/graph/constants.h index d3f1182909b5c..fa34ef75f2eb5 100644 --- a/include/onnxruntime/core/graph/constants.h +++ b/include/onnxruntime/core/graph/constants.h @@ -44,7 +44,6 @@ constexpr const char* kDmlExecutionProvider = "DmlExecutionProvider"; constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider"; constexpr const char* kAclExecutionProvider = "ACLExecutionProvider"; constexpr const char* kArmNNExecutionProvider = "ArmNNExecutionProvider"; -constexpr const char* kRocmExecutionProvider = "ROCMExecutionProvider"; constexpr const char* kCoreMLExecutionProvider = "CoreMLExecutionProvider"; constexpr const char* kJsExecutionProvider = "JsExecutionProvider"; constexpr const char* kSnpeExecutionProvider = "SNPEExecutionProvider"; diff --git a/include/onnxruntime/core/providers/resource.h b/include/onnxruntime/core/providers/resource.h index bd123e1cd41c2..8f9451ad11a4e 100644 --- a/include/onnxruntime/core/providers/resource.h +++ b/include/onnxruntime/core/providers/resource.h @@ -7,8 +7,8 @@ enum ResourceOffset { cpu_resource_offset = 0, cuda_resource_offset = 10000, dml_resource_offset = 20000, - rocm_resource_offset = 30000, + migraphx_resource_offset = 30000, // offsets for other ort eps custom_ep_resource_offset = 10000000, // offsets for customized eps -}; \ No newline at end of file +}; diff --git a/include/onnxruntime/core/providers/rocm/rocm_context.h b/include/onnxruntime/core/providers/rocm/rocm_context.h deleted file mode 100644 index aad1736217129..0000000000000 --- a/include/onnxruntime/core/providers/rocm/rocm_context.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#define ORT_ROCM_CTX - -#include "rocm_resource.h" -#include "core/providers/custom_op_context.h" -#include -#include -#include - -namespace Ort { - -namespace Custom { - -struct RocmContext : public CustomOpContext { - hipStream_t hip_stream = {}; - miopenHandle_t miopen_handle = {}; - hipblasHandle_t blas_handle = {}; - - void Init(const OrtKernelContext& kernel_ctx) { - const auto& ort_api = Ort::GetApi(); - void* resource = {}; - OrtStatus* status = nullptr; - - status = ort_api.KernelContext_GetResource( - &kernel_ctx, ORT_ROCM_RESOURCE_VERSION, RocmResource::hip_stream_t, &resource); - if (status) { - ORT_CXX_API_THROW("failed to fetch hip stream", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - hip_stream = reinterpret_cast(resource); - - resource = {}; - status = ort_api.KernelContext_GetResource( - &kernel_ctx, ORT_ROCM_RESOURCE_VERSION, RocmResource::miopen_handle_t, &resource); - if (status) { - ORT_CXX_API_THROW("failed to fetch miopen handle", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - miopen_handle = reinterpret_cast(resource); - - resource = {}; - status = ort_api.KernelContext_GetResource( - &kernel_ctx, ORT_ROCM_RESOURCE_VERSION, RocmResource::hipblas_handle_t, &resource); - if (status) { - ORT_CXX_API_THROW("failed to fetch hipblas handle", OrtErrorCode::ORT_RUNTIME_EXCEPTION); - } - blas_handle = reinterpret_cast(resource); - } -}; - -} // namespace Custom -} // namespace Ort diff --git a/include/onnxruntime/core/providers/rocm/rocm_resource.h b/include/onnxruntime/core/providers/rocm/rocm_resource.h deleted file mode 100644 index db032b48714c3..0000000000000 --- a/include/onnxruntime/core/providers/rocm/rocm_resource.h +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/resource.h" - -#define ORT_ROCM_RESOURCE_VERSION 1 - -enum RocmResource : int { - hip_stream_t = rocm_resource_offset, - miopen_handle_t, - hipblas_handle_t, - deferred_cpu_allocator_t, - // below are rocm ep options - device_id_t, // 10004 - arena_extend_strategy_t -}; diff --git a/include/onnxruntime/core/session/onnxruntime_lite_custom_op.h b/include/onnxruntime/core/session/onnxruntime_lite_custom_op.h index 5002e16ba116c..81c20768c3120 100644 --- a/include/onnxruntime/core/session/onnxruntime_lite_custom_op.h +++ b/include/onnxruntime/core/session/onnxruntime_lite_custom_op.h @@ -447,18 +447,6 @@ struct OrtLiteCustomOp : public OrtCustomOp { } #endif -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if::value, std::tuple>::type - CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { - thread_local RocmContext rocm_context; - rocm_context.Init(*context); - std::tuple current = std::tuple{rocm_context}; - auto next = CreateTuple(context, args, num_input, num_output, ep); - return std::tuple_cat(current, next); - } -#endif - template static typename std::enable_if::value, std::tuple>::type CreateTuple(OrtKernelContext* context, ArgPtrs& args, size_t num_input, size_t num_output, const std::string& ep) { @@ -674,14 +662,6 @@ struct OrtLiteCustomOp : public OrtCustomOp { } #endif -#ifdef ORT_ROCM_CTX - template - static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type - ParseArgs(std::vector& input_types, std::vector& output_types) { - ParseArgs(input_types, output_types); - } -#endif - template static typename std::enable_if<0 <= sizeof...(Ts) && std::is_same::value>::type ParseArgs(std::vector& input_types, std::vector& output_types) { diff --git a/java/src/test/java/ai/onnxruntime/InferenceTest.java b/java/src/test/java/ai/onnxruntime/InferenceTest.java index c202b2a9f80e0..71505f51efaca 100644 --- a/java/src/test/java/ai/onnxruntime/InferenceTest.java +++ b/java/src/test/java/ai/onnxruntime/InferenceTest.java @@ -2127,9 +2127,6 @@ private static SqueezeNetTuple openSessionSqueezeNet(EnumSet provid case ARM_NN: options.addArmNN(false); break; - case ROCM: - options.addROCM(); - break; case CORE_ML: options.addCoreML(); break; diff --git a/onnxruntime/contrib_ops/cpu/bert/multihead_attention_helper.h b/onnxruntime/contrib_ops/cpu/bert/multihead_attention_helper.h index 8af6faadd6e92..ba6da7284247f 100644 --- a/onnxruntime/contrib_ops/cpu/bert/multihead_attention_helper.h +++ b/onnxruntime/contrib_ops/cpu/bert/multihead_attention_helper.h @@ -346,12 +346,11 @@ Status CheckInputs(const T* query, // // The following inputs are not used in cross attention (so they are None for cross attention): // past_key : (B, N, P, H), or (B, N, M, H) when past_present_share_buffer is True. - // For CUDA, past_present_share_buffer is always True. ROCm supports both. + // For CUDA, past_present_share_buffer is always True. // past_value : (B, N, P, H), or (B, N, M, H) when past_present_share_buffer is True. - // For CUDA, past_present_share_buffer is always True. ROCm supports both. + // For CUDA, past_present_share_buffer is always True. // past_sequence_length : scalar (1) when past_present_share_buffer is True. // CUDA version has extra inputs (beam_width, cache_indirection) that are not checked in the class. - // For ROCm, see contrib_ops/rocm/bert/batched_gemm_softmax_gemm_permute_pipelines.cuh for more details. // --------------------------------------------------------------- AttentionQkvFormat qkv_format = UNKNOWN; diff --git a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc index 537d066b264a1..4597b9c7d6605 100644 --- a/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc +++ b/onnxruntime/contrib_ops/cpu/transformers/subgraph_base.cc @@ -132,8 +132,7 @@ const IExecutionProvider* Subgraph::GetProvider() const { const ExecutionProviders& providers = session_state_->GetExecutionProviders(); const IExecutionProvider* cpu_provider = providers.Get(onnxruntime::kCpuExecutionProvider); const IExecutionProvider* cuda_provider = providers.Get(onnxruntime::kCudaExecutionProvider); - const IExecutionProvider* rocm_provider = providers.Get(onnxruntime::kRocmExecutionProvider); - const IExecutionProvider* gpu_provider = cuda_provider ? cuda_provider : rocm_provider; + const IExecutionProvider* gpu_provider = cuda_provider; const IExecutionProvider* provider = gpu_provider ? gpu_provider : cpu_provider; return provider; } diff --git a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu index 0c4d75aeddac0..0e97d5387c0a5 100644 --- a/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu +++ b/onnxruntime/contrib_ops/cuda/bert/add_bias_transpose.cu @@ -216,7 +216,6 @@ __global__ void AddBiasTransposeQKV(int M, const T* input, const T* biases, T* o } } -#ifndef USE_ROCM template __global__ void AddBiasTransposeQKV(int M, const T* input, const T* biases, T* output, T* qkv_add_bias, const int rotary_embedding_dim, const int head_size, const int step, @@ -359,7 +358,6 @@ __global__ void AddBiasTransposeQKV(int M, const T* input, const T* biases, T* o } } } -#endif // this suppose 3 matrix in total template @@ -677,9 +675,7 @@ void InvokeAddBiasTranspose( assert(num_heads <= max_threads_per_block); if (do_rotary) { -#ifdef USE_ROCM - ORT_THROW("Rotary Attention is not supported on ROCm"); -#elif !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530 +#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530 if (format != 1 && format != 2 && format != 3) { ORT_THROW("format must be 1, 2 or 3 for rotary attention"); } diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu index bc5f4871283bb..985d81d558716 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_impl.cu @@ -787,8 +787,6 @@ Status UnfusedAttention( return result; } -#ifndef USE_ROCM // exclude the following from hipify since they are not used in ROCM EP - template Status ConcatPastToPresent(int batch_size, int num_heads, int qk_head_size, int v_head_size, int sequence_length, int total_sequence_length, @@ -859,7 +857,6 @@ template Status ConcatPastToPresent(int batch_size, int num_heads, int qk_ cudaStream_t stream, int max_threads_per_block, AttentionData& data); -#endif template Status PastPresentBufferShare(int batch_size, int num_heads, int qk_head_size, int v_head_size, diff --git a/onnxruntime/contrib_ops/cuda/bert/attention_kv_cache.cu b/onnxruntime/contrib_ops/cuda/bert/attention_kv_cache.cu index 80152e918ae30..84f651ca5470d 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention_kv_cache.cu +++ b/onnxruntime/contrib_ops/cuda/bert/attention_kv_cache.cu @@ -250,8 +250,6 @@ Status LaunchConcatTensorToTensor(cudaStream_t stream, return CUDA_CALL(cudaGetLastError()); } -#ifndef USE_ROCM // exclude the following from hipify since they are not used in ROCM EP - // ---------------------------------------------------------------------------------- // Below kernels are for past and present sharing buffer // ---------------------------------------------------------------------------------- @@ -397,7 +395,6 @@ template Status LaunchAddBiasTransAppendKvToPresent(cudaStream_t stream, const BFloat16* bias, const BFloat16* qkv_buffer, BFloat16* present); -#endif // Kernel to append new and past kv in either BSNH or BNSH format // Adapted from ConcatTensorToTensor kernel in attention_kv_cache.cu file diff --git a/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc b/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc index ee49f362564a6..5e5f909415fff 100644 --- a/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc +++ b/onnxruntime/contrib_ops/cuda/bert/decoder_attention.cc @@ -370,11 +370,7 @@ Status DecoderAttention::ComputeInternal(OpKernelContext* context) const { return LaunchDecoderAttentionKernel( device_prop, -#ifdef USE_ROCM - GetTuningContext(), -#else UseTF32(), -#endif context->GetComputeStream(), cublas, element_size, diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc index 3a16f16466ed3..e7ed96d7f5ee2 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.cc @@ -6,11 +6,7 @@ #include "fast_gelu.h" #include "core/providers/cuda/tensor/gelu_impl.h" #include "contrib_ops/cpu/bert/bias_gelu_helper.h" -#ifdef USE_ROCM -#include "contrib_ops/rocm/bert/elementwise.h" -#else #include "contrib_ops/cuda/bert/transformer_common.h" -#endif namespace onnxruntime { namespace contrib { @@ -36,10 +32,8 @@ using namespace ONNX_NAMESPACE; template FastGelu::FastGelu(const OpKernelInfo& op_kernel_info) : CudaKernel(op_kernel_info) { -#ifndef USE_ROCM const TransformerOptions* options = TransformerOptions::GetInstance(); use_half2_ = !options->DisableHalf2(); -#endif } template @@ -57,13 +51,6 @@ Status FastGelu::ComputeInternal(OpKernelContext* context) const { int64_t bias_length = (nullptr == bias) ? 0 : bias->Shape().Size(); typedef typename ToCudaType::MappedType CudaT; -#ifdef USE_ROCM - return LaunchElementwiseKernel( - GetTuningContext(), context->GetComputeStream(), - reinterpret_cast(input->Data()), static_cast(input_length), - (nullptr != bias) ? reinterpret_cast(bias->Data()) : nullptr, static_cast(bias_length), - reinterpret_cast(output->MutableData())); -#else return LaunchFastGeluKernel(GetDeviceProp(), Stream(context), static_cast(input_length), @@ -72,7 +59,6 @@ Status FastGelu::ComputeInternal(OpKernelContext* context) const { (nullptr != bias) ? reinterpret_cast(bias->Data()) : nullptr, reinterpret_cast(output->MutableData()), use_half2_); -#endif } } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h index 26f3bd5a03928..3e642a70afef5 100644 --- a/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h +++ b/onnxruntime/contrib_ops/cuda/bert/fast_gelu.h @@ -18,9 +18,7 @@ class FastGelu final : public CudaKernel { Status ComputeInternal(OpKernelContext* ctx) const override; private: -#ifndef USE_ROCM bool use_half2_; -#endif }; } // namespace cuda diff --git a/onnxruntime/contrib_ops/cuda/bert/rotary_embedding_util.h b/onnxruntime/contrib_ops/cuda/bert/rotary_embedding_util.h index 320aa2a552198..9238dde012c3c 100644 --- a/onnxruntime/contrib_ops/cuda/bert/rotary_embedding_util.h +++ b/onnxruntime/contrib_ops/cuda/bert/rotary_embedding_util.h @@ -27,8 +27,6 @@ using namespace onnxruntime::cuda; namespace onnxruntime { namespace cuda { -#ifndef USE_ROCM - inline __device__ float2 rotary_embedding_coefficient(const int zid, const int rot_embed_dim, const float t_step) { const float inv_freq = t_step / pow(10000.0f, zid / (float)rot_embed_dim); return {cos(inv_freq), sin(inv_freq)}; @@ -422,7 +420,5 @@ __device__ __inline__ void write_smem_transpose(const float2& vec, float* smem, smem[smem_pitch + transpose_idx] = vec.y; } -#endif - } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/bert/utils.cuh b/onnxruntime/contrib_ops/cuda/bert/utils.cuh index a45664083f5c7..83c853548abda 100644 --- a/onnxruntime/contrib_ops/cuda/bert/utils.cuh +++ b/onnxruntime/contrib_ops/cuda/bert/utils.cuh @@ -77,8 +77,6 @@ struct Float4_ { float2 y; }; -#ifndef USE_ROCM - template struct num_elems; template <> @@ -935,7 +933,5 @@ inline __device__ void ConvertFromFloat(uint4& dst, Float8_ src) { dst.w = Float2ToHalf2(src.w); } -#endif - } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.cc b/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.cc index feb6613690c08..f421a0db5a2f9 100644 --- a/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.cc +++ b/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.cc @@ -259,7 +259,6 @@ Status AllReduce::ComputeInternal(OpKernelContext* context) const { void* output_data = context->Output(0, in_shape)->MutableDataRaw(); -#ifndef USE_ROCM return FuncCustomAllReduce(nccl_, Stream(context), input_data, @@ -267,12 +266,6 @@ Status AllReduce::ComputeInternal(OpKernelContext* context) const { input_count, input_tensor->DataType(), onnxruntime::cuda::collective::IPCMemoryResourcePack::GetGlobalInstance()); -#else - ncclComm_t comm = nccl_->Comm(); - ncclDataType_t dtype = GetNcclDataType(input_tensor->DataType()); - NCCL_RETURN_IF_ERROR(ncclAllReduce(input_data, output_data, input_count, dtype, ncclSum, comm, Stream(context))); - return Status::OK(); -#endif } AllGather::AllGather(const OpKernelInfo& info) : NcclKernel(info) { @@ -428,7 +421,6 @@ Status FuncAllReduce( return Status::OK(); } -#ifndef USE_ROCM Status FuncCustomAllReduce( NcclContext* nccl, cudaStream_t stream, @@ -478,7 +470,6 @@ Status FuncCustomAllReduce( return Status::OK(); } -#endif static std::vector CalculatePermToSwapAxes( const int64_t axis, diff --git a/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.h b/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.h index 49646637b635e..8dac48492cc12 100644 --- a/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.h +++ b/onnxruntime/contrib_ops/cuda/collective/nccl_kernels.h @@ -6,11 +6,9 @@ #include "core/providers/cuda/cuda_kernel.h" #if defined(ORT_USE_NCCL) || defined(USE_MPI) -#ifndef USE_ROCM #include "custom_reduce_impl.h" #include "ipc_utils.h" #endif -#endif #if defined(ORT_USE_NCCL) #include @@ -107,7 +105,6 @@ Status FuncAllReduce( const Tensor* input, Tensor* output); -#ifndef USE_ROCM Status FuncCustomAllReduce( NcclContext* nccl, cudaStream_t stream, @@ -116,7 +113,6 @@ Status FuncCustomAllReduce( int64_t input_count, onnxruntime::MLDataType data_type, onnxruntime::cuda::collective::IPCMemoryResourcePack& ipc_mem_res_pack); -#endif void FuncAllGather( const NcclKernel* nccl_kernel, diff --git a/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu b/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu index 58c94f966841b..f12e6c530ff35 100644 --- a/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu +++ b/onnxruntime/contrib_ops/cuda/diffusion/group_norm_impl.cu @@ -128,7 +128,6 @@ Status LaunchGroupNormKernel( bool use_silu, bool broadcast_skip, int channels_per_block) { - // tuning_ctx only used for ROCm EP. ORT_UNUSED_PARAMETER(tuning_ctx); GroupNormNHWCParams params(output, add_out, input, skip, bias, gamma, beta, reinterpret_cast(workspace), epsilon, diff --git a/onnxruntime/contrib_ops/cuda/math/bias_gelu_impl.cu b/onnxruntime/contrib_ops/cuda/math/bias_gelu_impl.cu index 5a13520254e6d..35bfc111c6492 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_gelu_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/bias_gelu_impl.cu @@ -14,11 +14,7 @@ namespace cuda { namespace { constexpr int kElementsPerThread = GridDim::maxElementsPerThread; -#ifdef USE_ROCM -constexpr int kThreadsPerBlock = 512; -#else constexpr int kThreadsPerBlock = GridDim::maxThreadsPerBlock; -#endif } // namespace diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc index a95965775484d..db5e0e30af46d 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax.cc @@ -31,12 +31,7 @@ struct DispatchBiasSoftmaxImpl { } // namespace -// MIOpen doesn't support double so ROCm kernel doesn't have double support for now. -#ifdef USE_ROCM -#define BIAS_SOFTMAX_TYPES float, MLFloat16 -#else #define BIAS_SOFTMAX_TYPES float, MLFloat16, double -#endif ONNX_OPERATOR_KERNEL_EX( BiasSoftmax, kMSDomain, 1, kCudaExecutionProvider, diff --git a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu index 427c7fc624309..e665c35269b6f 100644 --- a/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/bias_softmax_impl.cu @@ -41,11 +41,7 @@ __global__ void BiasSoftmaxWarpForward(output_t* output, const input_t* input, c constexpr int next_power_of_two = 1 << log2_elements; constexpr int WARP_SIZE = next_power_of_two < GPU_WARP_SIZE ? next_power_of_two : GPU_WARP_SIZE; constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; -#ifdef USE_ROCM - constexpr int WARP_BATCH = 1; -#else constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; -#endif // each "WARP" (<=32) processes WARP_BATCH(one of {1,2}) batches int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; @@ -137,13 +133,8 @@ Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* outpu int warp_size = std::min(next_power_of_two, GPU_WARP_SIZE_HOST); // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_forward. -#ifdef USE_ROCM - int batches_per_warp = 1; - constexpr int threads_per_block = 256; -#else int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; constexpr int threads_per_block = 128; -#endif int warps_per_block = (threads_per_block / warp_size); int batches_per_block = warps_per_block * batches_per_warp; @@ -229,7 +220,7 @@ Status BiasSoftmaxImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* outpu const T* input_data, const T* bias_data, int element_count, int batch_count, \ bool is_inner_broadcast, int bias_broadcast_size); -// MIOpen doesn't support double so ROCm kernel doesn't have double support for now. +// MIOpen doesn't support double for now. SPECIALIZED_BIAS_SOFTMAX_IMPL(float) SPECIALIZED_BIAS_SOFTMAX_IMPL(half) #ifdef USE_CUDA diff --git a/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc b/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc index bec78d081ef69..afdd25a617ce3 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc +++ b/onnxruntime/contrib_ops/cuda/transformers/beam_search.cc @@ -70,9 +70,7 @@ BeamSearch::BeamSearch(const OpKernelInfo& info) GenerationCudaDeviceHelper::InitBeamState, GenerationCudaDeviceHelper::CreateBeamScorer); -#ifndef USE_ROCM SetDeviceHelpers_Cuda(GenerationCudaDeviceHelper::ReorderPastState, GenerationCudaDeviceHelper::InitCacheIndir); -#endif SetDeviceHelpers_Gpt(GenerationCudaDeviceHelper::UpdateGptFeeds, GenerationCudaDeviceHelper::UpdateGptFeeds); @@ -87,12 +85,10 @@ BeamSearch::BeamSearch(const OpKernelInfo& info) SetConsoleDumper(&g_cuda_dumper); -#ifndef USE_ROCM cuda_device_prop_ = &reinterpret_cast(info.GetExecutionProvider())->GetDeviceProp(); cuda_device_arch_ = static_cast(cuda_device_prop_)->major * 100 + static_cast(cuda_device_prop_)->minor * 10; -#endif } Status BeamSearch::ComputeInternal(OpKernelContext* context) const { @@ -124,9 +120,7 @@ WhisperBeamSearch::WhisperBeamSearch(const OpKernelInfo& info) GenerationCudaDeviceHelper::InitBeamState, GenerationCudaDeviceHelper::CreateBeamScorer); -#ifndef USE_ROCM SetDeviceHelpers_Cuda(GenerationCudaDeviceHelper::ReorderPastState, GenerationCudaDeviceHelper::InitCacheIndir); -#endif SetDeviceHelpers_Gpt(GenerationCudaDeviceHelper::UpdateGptFeeds, GenerationCudaDeviceHelper::UpdateGptFeeds); @@ -141,12 +135,10 @@ WhisperBeamSearch::WhisperBeamSearch(const OpKernelInfo& info) SetConsoleDumper(&g_cuda_dumper); -#ifndef USE_ROCM cuda_device_prop_ = &reinterpret_cast(info.GetExecutionProvider())->GetDeviceProp(); cuda_device_arch_ = static_cast(cuda_device_prop_)->major * 100 + static_cast(cuda_device_prop_)->minor * 10; -#endif } Status WhisperBeamSearch::ComputeInternal(OpKernelContext* context) const { diff --git a/onnxruntime/contrib_ops/cuda/transformers/beam_search_topk.cu b/onnxruntime/contrib_ops/cuda/transformers/beam_search_topk.cu index 44be2ef2375ee..dee9f9a95abcb 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/beam_search_topk.cu +++ b/onnxruntime/contrib_ops/cuda/transformers/beam_search_topk.cu @@ -226,11 +226,9 @@ void TopKLauncherMaxK( dim3 grid(batch_size * num_beams, voc_parts); -#ifndef USE_ROCM cudaFuncSetAttribute(BeamSearchOnlineTopKStage1Kernel, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1); -#endif // !USE_ROCM BeamSearchOnlineTopKStage1Kernel <<>>(input, K, vocab_size, (vocab_size + voc_parts - 1) / voc_parts, output_values_tmp, output_indices_tmp); diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_cuda_impl.cu b/onnxruntime/contrib_ops/cuda/transformers/generation_cuda_impl.cu index 94cc13e4b3b1c..52614a81d623f 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/generation_cuda_impl.cu +++ b/onnxruntime/contrib_ops/cuda/transformers/generation_cuda_impl.cu @@ -1263,7 +1263,6 @@ void UpdateDecoderMaskedMultiHeadAttentionCacheIndirection(int32_t* tgt_indir_ca current_length); } -#ifndef USE_ROCM namespace { template struct TypeMapper : public V_vec_m_ {}; @@ -1278,7 +1277,6 @@ struct TypeMapper { using Type = uint4; }; } // namespace -#endif template __global__ void KeyCacheExpansionKernel(const T* input, @@ -1330,7 +1328,6 @@ void KeyCacheExpansionKernelLauncher(const T* key_cache, tpb |= (tpb >> 16); tpb++; -#ifndef USE_ROCM if ((head_size % 4) == 0) { using vec_type = typename TypeMapper::Type; const dim3 block(tpb); @@ -1348,16 +1345,13 @@ void KeyCacheExpansionKernelLauncher(const T* key_cache, max_seq_length, equiv_head_size); } else { -#endif const dim3 block(tpb); KeyCacheExpansionKernel<<>>(key_cache, key_cache_expanded, beam_width, max_seq_length, head_size); -#ifndef USE_ROCM } -#endif } template void KeyCacheExpansionKernelLauncher(const float* key_cache, @@ -1417,7 +1411,6 @@ void BufferExpansionKernelLauncher(const T* input, cudaStream_t stream) { const dim3 block(128); -#ifndef USE_ROCM if ((chunk_size % 4) == 0) { using vec_type = typename TypeMapper::Type; const dim3 grid(batch_size, beam_width, (chunk_size / 4 + block.x - 1) / block.x); @@ -1431,14 +1424,11 @@ void BufferExpansionKernelLauncher(const T* input, reinterpret_cast(output), chunk_size / 2); } else { -#endif const dim3 grid(batch_size, beam_width, (chunk_size + block.x - 1) / block.x); BufferExpansionKernel<<>>(input, output, chunk_size); -#ifndef USE_ROCM } -#endif } template void BufferExpansionKernelLauncher(const float* input, diff --git a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc index d20d0b4218bd3..a3781c8e6cfa3 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc +++ b/onnxruntime/contrib_ops/cuda/transformers/generation_device_helper.cc @@ -21,7 +21,6 @@ #include "contrib_ops/cuda/transformers/greedy_search_top_one.h" #include "core/providers/cuda/tensor/transpose.h" -// the includes would be dummy for ROCm, we will ignore them for now #ifdef ENABLE_NVTX_PROFILE #include "core/providers/cuda/nvtx_profile.h" #include "core/providers/cuda/nvtx_profile_context.h" diff --git a/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc b/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc index cf623ab36015e..69756684b0c32 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc +++ b/onnxruntime/contrib_ops/cuda/transformers/greedy_search.cc @@ -39,21 +39,17 @@ GreedySearch::GreedySearch(const OpKernelInfo& info) GenerationCudaDeviceHelper::InitGreedyState, GenerationCudaDeviceHelper::InitGreedyState); -#ifndef USE_ROCM SetDeviceHelpers_Cuda(GenerationCudaDeviceHelper::ReorderPastState); -#endif SetDeviceHelpers_Gpt(GenerationCudaDeviceHelper::UpdateGptFeeds, GenerationCudaDeviceHelper::UpdateGptFeeds); SetConsoleDumper(&g_cuda_dumper_greedysearch); -#ifndef USE_ROCM cuda_device_prop_ = &reinterpret_cast(info.GetExecutionProvider())->GetDeviceProp(); cuda_device_arch_ = static_cast(cuda_device_prop_)->major * 100 + static_cast(cuda_device_prop_)->minor * 10; -#endif } Status GreedySearch::ComputeInternal(OpKernelContext* context) const { diff --git a/onnxruntime/contrib_ops/cuda/transformers/sampling.cc b/onnxruntime/contrib_ops/cuda/transformers/sampling.cc index a9cbdfd324ad7..c61ef36529174 100644 --- a/onnxruntime/contrib_ops/cuda/transformers/sampling.cc +++ b/onnxruntime/contrib_ops/cuda/transformers/sampling.cc @@ -40,21 +40,17 @@ Sampling::Sampling(const OpKernelInfo& info) GenerationCudaDeviceHelper::InitGreedyState, GenerationCudaDeviceHelper::InitGreedyState); -#ifndef USE_ROCM SetDeviceHelpers_Cuda(GenerationCudaDeviceHelper::ReorderPastState); -#endif SetDeviceHelpers_Gpt(GenerationCudaDeviceHelper::UpdateGptFeeds, GenerationCudaDeviceHelper::UpdateGptFeeds); SetConsoleDumper(&g_cuda_dumper_sampling); -#ifndef USE_ROCM gpu_device_prop_ = &reinterpret_cast(info.GetExecutionProvider())->GetDeviceProp(); gpu_device_arch_ = static_cast(gpu_device_prop_)->major * 100 + static_cast(gpu_device_prop_)->minor * 10; -#endif } Status Sampling::ComputeInternal(OpKernelContext* context) const { diff --git a/onnxruntime/core/framework/session_state.cc b/onnxruntime/core/framework/session_state.cc index 8fb3dc63aa4d1..a14e219d9c039 100644 --- a/onnxruntime/core/framework/session_state.cc +++ b/onnxruntime/core/framework/session_state.cc @@ -1179,7 +1179,7 @@ Status SessionState::CreateSubgraphSessionState() { const auto& ep = node.GetExecutionProviderType(); if (!ep.empty() && ep != kCpuExecutionProvider && ep != kCudaExecutionProvider && - ep != kRocmExecutionProvider && ep != kDmlExecutionProvider && + ep != kDmlExecutionProvider && ep != kJsExecutionProvider && ep != kWebGpuExecutionProvider) { // SessionState is only used when ORT is executing the subgraph. If a non-ORT EP has taken the control flow // node containing the subgraph it will create whatever state it needs internally. diff --git a/onnxruntime/core/optimizer/bias_softmax_fusion.cc b/onnxruntime/core/optimizer/bias_softmax_fusion.cc index 2bbc70db16cde..c37561c0086b0 100644 --- a/onnxruntime/core/optimizer/bias_softmax_fusion.cc +++ b/onnxruntime/core/optimizer/bias_softmax_fusion.cc @@ -44,7 +44,7 @@ bool TryBiasSoftmaxSubgraphMatch(Graph& graph, Node& start, Node*& add, Node*& s // check node is add and has single output if (!graph_utils::IsSupportedOptypeVersionAndDomain(node, "Add", {7, 13, 14}) || - !graph_utils::IsSupportedProvider(node, {kCudaExecutionProvider, kRocmExecutionProvider}) || + !graph_utils::IsSupportedProvider(node, {kCudaExecutionProvider}) || !optimizer_utils::CheckOutputEdges(graph, node, 1)) { return false; } @@ -239,7 +239,7 @@ Status BiasSoftmaxFusion::ApplyImpl(Graph& graph, bool& modified, int graph_leve // only support GPU execution provider auto& cep = GetCompatibleExecutionProviders(); - if (cep.size() > 0 && cep.find(kCudaExecutionProvider) == cep.end() && cep.find(kRocmExecutionProvider) == cep.end()) + if (cep.size() > 0 && cep.find(kCudaExecutionProvider) == cep.end()) return Status::OK(); for (auto node_index : node_topology_list) { diff --git a/onnxruntime/core/optimizer/conv_activation_fusion.cc b/onnxruntime/core/optimizer/conv_activation_fusion.cc index 04f74eb860443..b7f5af5888be0 100644 --- a/onnxruntime/core/optimizer/conv_activation_fusion.cc +++ b/onnxruntime/core/optimizer/conv_activation_fusion.cc @@ -79,7 +79,7 @@ class ConvActivationSelector : public NodeSelector { return std::nullopt; } - auto is_supported_non_cuda_rocm_ep_activation = [&graph_viewer](const Node& activation_node) { + auto is_supported_non_cuda_ep_activation = [&graph_viewer](const Node& activation_node) { if (graph_utils::IsSupportedOptypeVersionAndDomain(activation_node, "Relu", {6, 13, 14}) || graph_utils::IsSupportedOptypeVersionAndDomain(activation_node, "Sigmoid", {6, 13}) || graph_utils::IsSupportedOptypeVersionAndDomain(activation_node, "Tanh", {6, 13}) || @@ -105,17 +105,13 @@ class ConvActivationSelector : public NodeSelector { // check EP type and activation if (node_ep == kCudaExecutionProvider) { return std::nullopt; - } else if (node_ep == kRocmExecutionProvider) { - if (!graph_utils::IsSupportedOptypeVersionAndDomain(*next_node, "Relu", {6, 13, 14})) { - return std::nullopt; - } } else if (node_ep.empty() || node_ep == kCpuExecutionProvider || node_ep == kJsExecutionProvider || node_ep == kWebGpuExecutionProvider) { - if (!is_supported_non_cuda_rocm_ep_activation(*next_node) && + if (!is_supported_non_cuda_ep_activation(*next_node) && !graph_utils::IsSupportedOptypeVersionAndDomain(*next_node, "HardSigmoid", {6})) { return std::nullopt; } } else { - if (!is_supported_non_cuda_rocm_ep_activation(*next_node)) { + if (!is_supported_non_cuda_ep_activation(*next_node)) { return std::nullopt; } } diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 3680127ed4793..fdd4f5aa27862 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -76,7 +76,6 @@ #include "core/optimizer/quick_gelu_fusion.h" #include "core/optimizer/relu_clip_fusion.h" #include "core/optimizer/reshape_fusion.h" -#include "core/optimizer/rocm_blas_alt_impl.h" #include "core/optimizer/rule_based_graph_transformer.h" #include "core/optimizer/skip_layer_norm_fusion.h" #include "core/optimizer/slice_elimination.h" @@ -275,10 +274,6 @@ InlinedVector> GenerateTransformers( transformers.emplace_back(std::make_unique()); } - // add __backwardpass attribute to nodes after YieldOp, ROCm-only - const InlinedHashSet rocm_ep = {onnxruntime::kRocmExecutionProvider}; - transformers.emplace_back(std::make_unique(rocm_ep)); - // run TransposeOptimizer last as it works in a slightly different way by moving Transpose nodes around. // shouldn't affect the end result - just easier to debug any issue if it's last. transformers.emplace_back(std::make_unique(std::move(cpu_allocator))); @@ -305,33 +300,26 @@ InlinedVector> GenerateTransformers( const InlinedHashSet cuda_eps = {onnxruntime::kCudaExecutionProvider}; - const InlinedHashSet cuda_rocm_eps = {onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider}; - const InlinedHashSet cpu_cuda_rocm_eps = {onnxruntime::kCpuExecutionProvider, - onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider}; - const InlinedHashSet cpu_cuda_dml_rocm_eps = {onnxruntime::kCpuExecutionProvider, - onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider, - onnxruntime::kDmlExecutionProvider}; - const InlinedHashSet cpu_acl_cuda_dml_rocm_eps = {onnxruntime::kCpuExecutionProvider, - onnxruntime::kAclExecutionProvider, - onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider, - onnxruntime::kDmlExecutionProvider}; - const InlinedHashSet cpu_rocm_acl_armnn_js_webgpu_eps = {onnxruntime::kCpuExecutionProvider, - onnxruntime::kRocmExecutionProvider, + const InlinedHashSet cpu_cuda_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kCudaExecutionProvider}; + const InlinedHashSet cpu_cuda_dml_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kCudaExecutionProvider, + onnxruntime::kDmlExecutionProvider}; + const InlinedHashSet cpu_acl_cuda_dml_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kAclExecutionProvider, + onnxruntime::kCudaExecutionProvider, + onnxruntime::kDmlExecutionProvider}; + const InlinedHashSet cpu_acl_armnn_js_webgpu_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kAclExecutionProvider, + onnxruntime::kArmNNExecutionProvider, + onnxruntime::kJsExecutionProvider, + onnxruntime::kWebGpuExecutionProvider}; + const InlinedHashSet cpu_cuda_acl_armnn_js_webgpu_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kCudaExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider, onnxruntime::kJsExecutionProvider, onnxruntime::kWebGpuExecutionProvider}; - const InlinedHashSet cpu_cuda_rocm_acl_armnn_js_webgpu_eps = {onnxruntime::kCpuExecutionProvider, - onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider, - onnxruntime::kAclExecutionProvider, - onnxruntime::kArmNNExecutionProvider, - onnxruntime::kJsExecutionProvider, - onnxruntime::kWebGpuExecutionProvider}; const InlinedHashSet cpu_dml_acl_eps = {onnxruntime::kCpuExecutionProvider, onnxruntime::kDmlExecutionProvider, onnxruntime::kAclExecutionProvider}; @@ -362,30 +350,30 @@ InlinedVector> GenerateTransformers( transformers.emplace_back(std::make_unique(cpu_dml_acl_eps)); transformers.emplace_back(std::make_unique(cpu_acl_eps)); - transformers.emplace_back(std::make_unique(cpu_rocm_acl_armnn_js_webgpu_eps)); - - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps, level)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps, level)); - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_cuda_dml_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_armnn_js_webgpu_eps)); + + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps, level)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps, level)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_dml_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); transformers.emplace_back(std::make_unique(cuda_eps)); // Run MatMulAddFusion again after *AttentionFusion transforms with `preserve_attention_pattern = false`, // to cleanup the remaining MatMul-Add that were part of the attention pattern but not detected or fused. transformers.emplace_back(std::make_unique(no_limit_empty_ep_list, false)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_cuda_dml_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_dml_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); // GeluApproximation has side effects which may change results. It needs to be manually enabled, // or alternatively the model can be updated offline using a model conversion script // e.g. fusion_gelu_approximation function used by onnxruntime/python/tools/transformers/onnx_model_bert.py if (enable_gelu_approximation) { - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); } #ifdef ENABLE_TRITON @@ -396,15 +384,15 @@ InlinedVector> GenerateTransformers( } #endif // ENABLE_TRITON - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cuda_rocm_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); + transformers.emplace_back(std::make_unique(cuda_eps)); #ifdef ENABLE_TRAINING - transformers.emplace_back(std::make_unique(cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cuda_rocm_eps)); - transformers.emplace_back(std::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(std::make_unique(cuda_eps)); + transformers.emplace_back(std::make_unique(cuda_eps)); + transformers.emplace_back(std::make_unique(cpu_cuda_eps)); #endif - transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_rocm_eps)); + transformers.emplace_back(std::make_unique(cpu_acl_cuda_dml_eps)); transformers.emplace_back(std::make_unique(dml_ep)); #ifdef MLAS_TARGET_AMD64_IX86 diff --git a/onnxruntime/core/optimizer/layer_norm_fusion.cc b/onnxruntime/core/optimizer/layer_norm_fusion.cc index 1e88ed44b1a8a..8a7f83e871768 100644 --- a/onnxruntime/core/optimizer/layer_norm_fusion.cc +++ b/onnxruntime/core/optimizer/layer_norm_fusion.cc @@ -633,9 +633,7 @@ Status SimplifiedLayerNormFusion::ApplyImpl(Graph& graph, bool& modified, int gr // if there is a Cast between x and y. Having Cast between means cannot fuse. const Node* p_pow_input_node = graph_utils::GetInputNode(pow_node, 0); bool has_leading_cast = false; - bool is_gpu_ep = (pow_node.GetExecutionProviderType() == kCudaExecutionProvider || - pow_node.GetExecutionProviderType() == kRocmExecutionProvider) || - skip_device_check_; + bool is_gpu_ep = pow_node.GetExecutionProviderType() == kCudaExecutionProvider || skip_device_check_; if (is_gpu_ep && p_pow_input_node) { Node& pow_input_node = *graph.GetNode(p_pow_input_node->Index()); // If input to Pow is a Cast, and the Cast has 2 consumers only (Pow, Div) diff --git a/onnxruntime/core/optimizer/matmul_scale_fusion.cc b/onnxruntime/core/optimizer/matmul_scale_fusion.cc index 7ceb61b4aabc5..cc222e5e342dc 100644 --- a/onnxruntime/core/optimizer/matmul_scale_fusion.cc +++ b/onnxruntime/core/optimizer/matmul_scale_fusion.cc @@ -198,7 +198,6 @@ bool IsMatMulInputTypeSupported(const Node& node) { // if no matching key is present, any data type is allowed static const InlinedHashMap> k_supported_data_types{ {kCudaExecutionProvider, {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}}, - {kRocmExecutionProvider, {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}}, {kCpuExecutionProvider, {"tensor(float)"}}, }; diff --git a/onnxruntime/core/optimizer/rocm_blas_alt_impl.cc b/onnxruntime/core/optimizer/rocm_blas_alt_impl.cc deleted file mode 100644 index decb25f565efe..0000000000000 --- a/onnxruntime/core/optimizer/rocm_blas_alt_impl.cc +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -#include - -#include "core/optimizer/initializer.h" -#include "core/optimizer/rocm_blas_alt_impl.h" -#include "core/graph/graph_utils.h" - -using namespace ONNX_NAMESPACE; -using namespace ::onnxruntime::common; -namespace onnxruntime { - -Status RocmBlasAltImpl::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { - GraphViewer graph_viewer(graph); - const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); - - bool is_backward_pass = false; - - for (auto node_index : node_topology_list) { - auto& node = *graph.GetNode(node_index); - - if (node.OpType() == "YieldOp") { - is_backward_pass = true; - } - - ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger)); - - if (is_backward_pass) { - node.AddAttribute(std::string("__backwardpass"), static_cast(1)); - modified = true; - } - } - - return Status::OK(); -} -} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/rocm_blas_alt_impl.h b/onnxruntime/core/optimizer/rocm_blas_alt_impl.h deleted file mode 100644 index 11744d0dac32b..0000000000000 --- a/onnxruntime/core/optimizer/rocm_blas_alt_impl.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/optimizer/graph_transformer.h" -#include "core/graph/graph_utils.h" - -namespace onnxruntime { - -class RocmBlasAltImpl : public GraphTransformer { - public: - RocmBlasAltImpl(const InlinedHashSet& compatible_execution_providers = {}) noexcept - : GraphTransformer("RocmBlasAltImpl", compatible_execution_providers) {} - - Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; -}; - -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/cpu_provider_shared.h b/onnxruntime/core/providers/cpu/cpu_provider_shared.h index 9e49f068c680c..15baf7309070d 100644 --- a/onnxruntime/core/providers/cpu/cpu_provider_shared.h +++ b/onnxruntime/core/providers/cpu/cpu_provider_shared.h @@ -38,7 +38,7 @@ struct ProviderHostCPU { virtual Status NonMaxSuppressionBase__PrepareCompute(OpKernelContext* ctx, PrepareContext& pc) = 0; virtual Status NonMaxSuppressionBase__GetThresholdsFromInputs(const PrepareContext& pc, int64_t& max_output_boxes_per_class, float& iou_threshold, float& score_threshold) = 0; -#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) // From cpu/tensor/size.h virtual Status Size__Compute(const Size* p, OpKernelContext* context) = 0; @@ -254,7 +254,7 @@ struct ProviderHostCPU { extern ProviderHostCPU& g_host_cpu; -#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) namespace GatherElements { inline Status ValidateInputShapes(const TensorShape& input_data_shape, const TensorShape& indices_shape, @@ -336,7 +336,7 @@ inline Status ExecuteTritonOpByFuncName(OpKernelContext* p_ctx, const std::strin } // namespace contrib #endif // ENABLE_TRITON -#endif // USE_CUDA || USE_CUDA_PROVIDER_INTERFACE || USE_ROCM +#endif // USE_CUDA || USE_CUDA_PROVIDER_INTERFACE #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cpu/math/matmul_helper.h b/onnxruntime/core/providers/cpu/math/matmul_helper.h index d7275ee324756..9da7509eea2c6 100644 --- a/onnxruntime/core/providers/cpu/math/matmul_helper.h +++ b/onnxruntime/core/providers/cpu/math/matmul_helper.h @@ -23,7 +23,7 @@ inline void TensorShapeCopyDims(const TensorShape& shape, int64_t* dims, size_t class MatMulComputeHelper { public: // fill_offsets is to control if to fill offsets here. - // For CUDA/ROCM kernel when we can use GemmStridedBatched, we don't need to fill the offsets. + // For CUDA kernel when we can use GemmStridedBatched, we don't need to fill the offsets. Status Compute(const TensorShape& orig_left_shape, const TensorShape& orig_right_shape, bool transa = false, bool transb = false, bool trans_batch_a = false, bool trans_batch_b = false, diff --git a/onnxruntime/core/providers/cpu/object_detection/non_max_suppression_helper.h b/onnxruntime/core/providers/cpu/object_detection/non_max_suppression_helper.h index 5cfd1ecee602a..e20e9ce0c81c2 100644 --- a/onnxruntime/core/providers/cpu/object_detection/non_max_suppression_helper.h +++ b/onnxruntime/core/providers/cpu/object_detection/non_max_suppression_helper.h @@ -10,11 +10,6 @@ #define ORT_DEVICE __device__ #define HelperMin(a, b) _Min(a, b) #define HelperMax(a, b) _Max(a, b) -#elif defined(__HIPCC__) -#include "core/providers/rocm/cu_inc/common.cuh" -#define ORT_DEVICE __host__ __device__ -#define HelperMin(a, b) _Min(a, b) -#define HelperMax(a, b) _Max(a, b) #else #include #define ORT_DEVICE @@ -50,8 +45,6 @@ struct SelectedIndex { #ifdef __NVCC__ namespace cuda { -#elif defined(__HIPCC__) -namespace rocm { #endif namespace nms_helpers { @@ -151,7 +144,5 @@ inline bool SuppressByIOU(const float* boxes_data, int64_t box_index1, int64_t b } // namespace nms_helpers #ifdef __NVCC__ } // namespace cuda -#elif defined(__HIPCC__) -} // namespace rocm #endif } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh index 1469f55f0bfda..9d84920f76df9 100644 --- a/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh +++ b/onnxruntime/core/providers/cuda/cu_inc/binary_elementwise_impl.cuh @@ -194,13 +194,8 @@ void BinaryElementWiseNoBroadcastImpl( if (count == 0) // special case where there's a dim value of 0 in the output shape return; -#ifdef USE_ROCM - const int num_elements_per_thread = 2; - const int num_threads_per_block = 512; -#else const int num_elements_per_thread = GridDim::maxElementsPerThread; const int num_threads_per_block = GridDim::maxThreadsPerBlock; -#endif int blocksPerGrid = static_cast(CeilDiv(count, num_threads_per_block * num_elements_per_thread)); #define FUNC_CALL(NumElemT) \ @@ -237,13 +232,8 @@ void _BinaryElementWiseImpl( if (count == 0) // special case where there's a dim value of 0 in the output shape return; -#ifdef USE_ROCM - const int num_elements_per_thread = 2; - const int num_threads_per_block = 512; -#else const int num_elements_per_thread = GridDim::maxElementsPerThread; const int num_threads_per_block = GridDim::maxThreadsPerBlock; -#endif int blocksPerGrid = static_cast(CeilDiv(count, num_threads_per_block * num_elements_per_thread)); NumElemT N = static_cast(count); diff --git a/onnxruntime/core/providers/cuda/cu_inc/elementwise_impl.cuh b/onnxruntime/core/providers/cuda/cu_inc/elementwise_impl.cuh index 07a65bd252304..dc5b62dfbedf2 100644 --- a/onnxruntime/core/providers/cuda/cu_inc/elementwise_impl.cuh +++ b/onnxruntime/core/providers/cuda/cu_inc/elementwise_impl.cuh @@ -8,13 +8,8 @@ namespace onnxruntime { namespace cuda { -#ifdef USE_ROCM -constexpr int kElementsPerThread = 2; -constexpr int kThreadsPerBlock = 512; -#else constexpr int kElementsPerThread = GridDim::maxElementsPerThread; constexpr int kThreadsPerBlock = GridDim::maxThreadsPerBlock; -#endif template __global__ void ElementwiseKernel(T* output_data, const FuncT functor, TIndex N) { diff --git a/onnxruntime/core/providers/cuda/cuda_provider_interface.cc b/onnxruntime/core/providers/cuda/cuda_provider_interface.cc index 6cd5368cb7341..9632ecba3d951 100644 --- a/onnxruntime/core/providers/cuda/cuda_provider_interface.cc +++ b/onnxruntime/core/providers/cuda/cuda_provider_interface.cc @@ -2,7 +2,6 @@ // Licensed under the MIT License. #include "core/session/onnxruntime_c_api.h" -#if !defined(USE_ROCM) namespace onnxruntime { struct Provider; @@ -16,5 +15,3 @@ ORT_API(onnxruntime::Provider*, GetProvider) { return reinterpret_cast(onnxruntime::GetProvider()); } } - -#endif diff --git a/onnxruntime/core/providers/cuda/math/topk_impl.cuh b/onnxruntime/core/providers/cuda/math/topk_impl.cuh index 0cbc848be971c..9c9ed73079701 100644 --- a/onnxruntime/core/providers/cuda/math/topk_impl.cuh +++ b/onnxruntime/core/providers/cuda/math/topk_impl.cuh @@ -485,7 +485,7 @@ Status TopKImpl(const CudaKernel* kernel, bool use_deterministic_compute, int64_t N, \ int64_t dimension) -// This file is causing excessive long compilation time in ROCm EP. Split all those compilations into multiple +// This file is causing excessive long compilation time. Split all those compilations into multiple // translation units to speed it up. TOPKIMPLE(TOPK_IMPL_TYPE); diff --git a/onnxruntime/core/providers/cuda/nn/layer_norm_impl.cu b/onnxruntime/core/providers/cuda/nn/layer_norm_impl.cu index 90b542beaaf26..887d11a49db46 100644 --- a/onnxruntime/core/providers/cuda/nn/layer_norm_impl.cu +++ b/onnxruntime/core/providers/cuda/nn/layer_norm_impl.cu @@ -436,10 +436,6 @@ void HostApplyLayerNorm( parallel_rows *= 2; } dim3 threads(warp_size, threads_y, 1); -#ifdef __HIP_PLATFORM_HCC__ - // Optimization for ROCm MI100 - threads.y = 1; -#endif const dim3 blocks(1, std::min(n1, maxGridY), 1); int nshared = threads.y > 1 ? threads.y * sizeof(U) + (threads.y / 2) * sizeof(U) : 0; diff --git a/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h index 5238c42387eb2..ec87bb86acdb4 100644 --- a/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h +++ b/onnxruntime/core/providers/cuda/shared_inc/cuda_utils.h @@ -53,11 +53,7 @@ void Fill(cudaStream_t stream, T* output, T value, int64_t count); */ template struct TArray { -#if defined(USE_ROCM) -#define TARRAY_CONSTRUCTOR_SPECIFIERS __host__ __device__ -#else #define TARRAY_CONSTRUCTOR_SPECIFIERS -#endif TARRAY_CONSTRUCTOR_SPECIFIERS TArray() = default; TARRAY_CONSTRUCTOR_SPECIFIERS TArray(const TArray&) = default; diff --git a/onnxruntime/core/providers/cuda/tensor/concat_impl.cu b/onnxruntime/core/providers/cuda/tensor/concat_impl.cu index 84e1e76fae8de..f369ead0f90c4 100644 --- a/onnxruntime/core/providers/cuda/tensor/concat_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/concat_impl.cu @@ -10,13 +10,8 @@ namespace onnxruntime { namespace cuda { namespace { -#ifdef USE_ROCM -constexpr int kNumElementsPerThread = 2; -constexpr int kNumThreadsPerBlock = 512; -#else constexpr int kNumElementsPerThread = GridDim::maxElementsPerThread; constexpr int kNumThreadsPerBlock = GridDim::maxThreadsPerBlock; -#endif } // namespace // concat dimension are same for all inputs diff --git a/onnxruntime/core/providers/cuda/tensor/gather_elements_impl.cu b/onnxruntime/core/providers/cuda/tensor/gather_elements_impl.cu index 81acb81be5025..2c4d5e6403dd5 100644 --- a/onnxruntime/core/providers/cuda/tensor/gather_elements_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/gather_elements_impl.cu @@ -14,11 +14,7 @@ namespace onnxruntime { namespace cuda { namespace { -#ifdef USE_ROCM -constexpr int kThreadsPerBlock = 256; -#else constexpr int kThreadsPerBlock = GPU_WARP_SIZE * 4; -#endif constexpr int kThreadWorkSize = 4; // General case to compute the input(for Gather)/output(for Scatter) and indices data offset given the thread ID diff --git a/onnxruntime/core/providers/cuda/tensor/slice_impl.cu b/onnxruntime/core/providers/cuda/tensor/slice_impl.cu index df392b45e9d5e..84021a99a8606 100644 --- a/onnxruntime/core/providers/cuda/tensor/slice_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/slice_impl.cu @@ -9,13 +9,8 @@ namespace onnxruntime { namespace cuda { namespace { -#ifdef USE_ROCM -constexpr int kNumElementsPerThread = 2; -constexpr int kNumThreadsPerBlock = 512; -#else constexpr int kNumElementsPerThread = GridDim::maxElementsPerThread; constexpr int kNumThreadsPerBlock = GridDim::maxThreadsPerBlock; -#endif } // namespace template diff --git a/onnxruntime/core/providers/cuda/tensor/split.cc b/onnxruntime/core/providers/cuda/tensor/split.cc index 52775b2e8be7a..ca82387600085 100644 --- a/onnxruntime/core/providers/cuda/tensor/split.cc +++ b/onnxruntime/core/providers/cuda/tensor/split.cc @@ -76,7 +76,6 @@ Status SplitKernel::ComputeInternal(OpKernelContext* ctx) const { auto input_dims = input_shape.GetDims(); auto output_dimensions{input_shape.AsShapeVector()}; -#ifndef USE_ROCM if (split_sizes.size() == 3 && ((axis + 1) == gsl::narrow_cast(input_shape.NumDimensions()))) { // we use (axis + 1) == num_dimensions to check if we are splitting on inner most axis. // only when split on inner axis and output size is 3, we can use Split3Inner. @@ -101,7 +100,6 @@ Status SplitKernel::ComputeInternal(OpKernelContext* ctx) const { output2->MutableDataRaw(), input_dims); } -#endif CudaAsyncBuffer output_ptr(this, num_outputs); gsl::span output_ptr_span = output_ptr.CpuSpan(); diff --git a/onnxruntime/core/providers/cuda/tensor/split_impl.cu b/onnxruntime/core/providers/cuda/tensor/split_impl.cu index 6c2cdfe029a08..e8d26d5757bc0 100644 --- a/onnxruntime/core/providers/cuda/tensor/split_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/split_impl.cu @@ -10,13 +10,8 @@ namespace onnxruntime { namespace cuda { namespace { -#ifdef USE_ROCM -constexpr int kNumElementsPerThread = 2; -constexpr int kNumThreadsPerBlock = 512; -#else constexpr int kNumElementsPerThread = GridDim::maxElementsPerThread; constexpr int kNumThreadsPerBlock = GridDim::maxThreadsPerBlock; -#endif } // namespace template @@ -157,7 +152,6 @@ Status SplitImpl(cudaStream_t stream, const size_t element_size, const int block return Status::OK(); } -#ifndef USE_ROCM template __global__ void _Split3InnerKernel(const int64_t size0_in_byte, const int64_t size1_in_byte, @@ -264,7 +258,6 @@ Status Split3Inner(cudaStream_t stream, const size_t element_size, const int64_t return Status::OK(); } -#endif } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/tile_impl.cu b/onnxruntime/core/providers/cuda/tensor/tile_impl.cu index e3ef2965c5577..aaf54d276684a 100644 --- a/onnxruntime/core/providers/cuda/tensor/tile_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/tile_impl.cu @@ -7,13 +7,8 @@ namespace onnxruntime { namespace cuda { -#ifdef USE_ROCM -constexpr int num_elements_per_thread = 2; -constexpr int num_threads_per_block = 512; -#else constexpr int num_elements_per_thread = GridDim::maxElementsPerThread; constexpr int num_threads_per_block = GridDim::maxThreadsPerBlock; -#endif template __global__ void _UnRolledTileKernel(const size_t shape_rank, const TArray fdm_input_shape, diff --git a/onnxruntime/core/providers/get_execution_providers.cc b/onnxruntime/core/providers/get_execution_providers.cc index 9ecabcad504b3..69fbbf19241df 100644 --- a/onnxruntime/core/providers/get_execution_providers.cc +++ b/onnxruntime/core/providers/get_execution_providers.cc @@ -50,14 +50,6 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] = true, #else false, -#endif - }, - { - kRocmExecutionProvider, -#ifdef USE_ROCM - true, -#else - false, #endif }, { diff --git a/onnxruntime/core/providers/migraphx/gpu_data_transfer.cc b/onnxruntime/core/providers/migraphx/gpu_data_transfer.cc index c9cd6e21b4eba..4787f6a80e959 100644 --- a/onnxruntime/core/providers/migraphx/gpu_data_transfer.cc +++ b/onnxruntime/core/providers/migraphx/gpu_data_transfer.cc @@ -5,8 +5,6 @@ #include "core/providers/migraphx/gpu_data_transfer.h" #include "core/providers/migraphx/migraphx_call.h" -// If you make change below, please also update onnxruntime/core/providers/rocm/gpu_data_transfer.cc - namespace onnxruntime { bool GPUDataTransfer::CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const { diff --git a/onnxruntime/core/providers/migraphx/migraphx_stream_handle.cc b/onnxruntime/core/providers/migraphx/migraphx_stream_handle.cc index 0baa8a1c67c67..f95a9f755a8bd 100644 --- a/onnxruntime/core/providers/migraphx/migraphx_stream_handle.cc +++ b/onnxruntime/core/providers/migraphx/migraphx_stream_handle.cc @@ -12,7 +12,7 @@ namespace onnxruntime { enum MIGraphXResource { - hip_stream_t = rocm_resource_offset + hip_stream_t = migraphx_resource_offset }; struct MIGraphXNotification : synchronize::Notification { diff --git a/onnxruntime/core/providers/provider_factory_creators.h b/onnxruntime/core/providers/provider_factory_creators.h index b0d850ca04841..97f80478f6f8c 100644 --- a/onnxruntime/core/providers/provider_factory_creators.h +++ b/onnxruntime/core/providers/provider_factory_creators.h @@ -9,7 +9,7 @@ // The functions are typically implemented in // onnxruntime/core/providers//_provider_factory.cc. // -// For execution providers that are built as separate libraries (CUDA, TensorRT, ROCm, MIGraphX, DNNL, OpenVINO) +// For execution providers that are built as separate libraries (CUDA, TensorRT, MIGraphX, DNNL, OpenVINO) // the functions are implemented in provider_bridge_ort.cc. #include "core/providers/cpu/cpu_provider_factory_creator.h" @@ -62,10 +62,6 @@ #include "core/providers/rknpu/rknpu_provider_factory_creator.h" #endif -#if defined(USE_ROCM) -#include "core/providers/rocm/rocm_provider_factory_creator.h" -#endif - #if defined(USE_QNN) || defined(USE_QNN_PROVIDER_INTERFACE) #include "core/providers/qnn/qnn_provider_factory_creator.h" #endif diff --git a/onnxruntime/core/providers/shared_library/provider_api.h b/onnxruntime/core/providers/shared_library/provider_api.h index 5be46cd480004..46f05ee40aa17 100644 --- a/onnxruntime/core/providers/shared_library/provider_api.h +++ b/onnxruntime/core/providers/shared_library/provider_api.h @@ -298,7 +298,6 @@ constexpr const char* kCannExecutionProvider = "CANNExecutionProvider"; constexpr const char* kDnnlExecutionProvider = "DnnlExecutionProvider"; constexpr const char* kOpenVINOExecutionProvider = "OpenVINOExecutionProvider"; constexpr const char* kVitisAIExecutionProvider = "VitisAIExecutionProvider"; -constexpr const char* kRocmExecutionProvider = "ROCMExecutionProvider"; constexpr const char* kTensorrtExecutionProvider = "TensorrtExecutionProvider"; constexpr const char* kNvTensorRTRTXExecutionProvider = "NvTensorRTRTXExecutionProvider"; constexpr const char* kMIGraphXExecutionProvider = "MIGraphXExecutionProvider"; @@ -318,9 +317,6 @@ std::unique_ptr CreateCUDAPinnedAllocator(int16_t device_id, const c std::unique_ptr CreateMIGraphXAllocator(int16_t device_id, const char* name); std::unique_ptr CreateMIGraphXPinnedAllocator(int16_t device_id, const char* name); -std::unique_ptr CreateROCMAllocator(int16_t device_id, const char* name); -std::unique_ptr CreateROCMPinnedAllocator(int16_t device_id, const char* name); - std::unique_ptr CreateGPUDataTransfer(); std::unordered_set GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph, diff --git a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc index 0e5df0026d2c0..5732984af29b4 100644 --- a/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc +++ b/onnxruntime/core/providers/shared_library/provider_bridge_provider.cc @@ -533,7 +533,7 @@ Status NonMaxSuppressionBase::GetThresholdsFromInputs(const PrepareContext& pc, Status GatherBase::PrepareForCompute(OpKernelContext* context, GatherBase::Prepare& p) const { return g_host_cpu.GatherBase__PrepareForCompute(this, context, reinterpret_cast(p)); } Status UnsqueezeBase::PrepareCompute(OpKernelContext* ctx, UnsqueezeBase::Prepare& p) const { return g_host_cpu.UnsqueezeBase__PrepareCompute(this, ctx, reinterpret_cast(p)); } -#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) bool TileOp::IsTileMemcpy(const TensorShape& input_shape, const int64_t* repeats, size_t rank, bool& is_batched_memcpy, size_t& num_of_elements_per_batch, size_t& num_of_copies_per_batch, size_t& num_of_batch_copies) { return g_host_cpu.TileOp__IsTileMemcpy(input_shape, repeats, rank, is_batched_memcpy, num_of_elements_per_batch, num_of_copies_per_batch, num_of_batch_copies); } diff --git a/onnxruntime/core/providers/shared_library/provider_interfaces.h b/onnxruntime/core/providers/shared_library/provider_interfaces.h index f1d545d0c6b17..786cf8ce09ada 100644 --- a/onnxruntime/core/providers/shared_library/provider_interfaces.h +++ b/onnxruntime/core/providers/shared_library/provider_interfaces.h @@ -201,19 +201,6 @@ struct ProviderHost { virtual std::unique_ptr CreateMIGraphXAllocator(int16_t device_id, const char* name) = 0; virtual std::unique_ptr CreateMIGraphXPinnedAllocator(int16_t device_id, const char* name) = 0; -#ifdef USE_ROCM - virtual std::unique_ptr CreateROCMAllocator(int16_t device_id, const char* name) = 0; - virtual std::unique_ptr CreateROCMPinnedAllocator(int16_t device_id, const char* name) = 0; - - virtual void rocm__Impl_Cast(void* stream, const int64_t* input_data, int32_t* output_data, size_t count) = 0; - virtual void rocm__Impl_Cast(void* stream, const int32_t* input_data, int64_t* output_data, size_t count) = 0; - virtual void rocm__Impl_Cast(void* stream, const double* input_data, float* output_data, size_t count) = 0; - virtual void rocm__Impl_Cast(void* stream, const float* input_data, double* output_data, size_t count) = 0; - - virtual Status RocmCall_false(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; - virtual void RocmCall_true(int retCode, const char* exprString, const char* libName, int successCode, const char* msg, const char* file, const int line) = 0; -#endif - virtual std::unordered_set GetCpuPreferredNodes(const onnxruntime::GraphViewer& graph, const IExecutionProvider::IKernelLookup& kernel_lookup, gsl::span tentative_nodes, diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index ab3932e7abfb4..98d6228b58e9b 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -162,7 +162,6 @@ static bool AreAllComputeNodesAssignedToCudaOrJsOrDmlEpWebGpuEp(const Graph& gra // Empty node provider means CPU EP if (!node_provider.empty() && !(node_provider == kCudaExecutionProvider || - node_provider == kRocmExecutionProvider || node_provider == kJsExecutionProvider || node_provider == kWebGpuExecutionProvider || node_provider == kDmlExecutionProvider) && @@ -2269,7 +2268,7 @@ common::Status InferenceSession::Initialize() { "Session initialization canceled due to user request."); } - // Currently graph capture is only considered by CUDA EP, TRT EP, ROCM EP and JS EP. + // Currently graph capture is only considered by CUDA EP, TRT EP and JS EP. // // Check for CUDA EP: // If the CUDA EP is part of the providers list for this session AND @@ -2289,16 +2288,9 @@ common::Status InferenceSession::Initialize() { // All the "compute" graph nodes have been assigned to the JS EP, // Then the JS EP is cached for triggering a ReplayGraph() in Run(). // - // Check for ROCM EP: - // If the ROCM EP is part of the providers list for this session AND - // The ROCM EP is configured to do a graph capture AND - // All the "compute" graph nodes have been assigned to the ROCM EP, - // Then the ROCM EP is cached for triggering a ReplayGraph() in Run(). - // std::vector graph_support_ep_list = { onnxruntime::kTensorrtExecutionProvider, onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider, onnxruntime::kJsExecutionProvider, onnxruntime::kWebGpuExecutionProvider, onnxruntime::kDmlExecutionProvider}; @@ -2321,7 +2313,6 @@ common::Status InferenceSession::Initialize() { } if (strcmp(target_ep->Type().c_str(), onnxruntime::kCudaExecutionProvider) == 0 || - strcmp(target_ep->Type().c_str(), onnxruntime::kRocmExecutionProvider) == 0 || strcmp(target_ep->Type().c_str(), onnxruntime::kJsExecutionProvider) == 0 || strcmp(target_ep->Type().c_str(), onnxruntime::kWebGpuExecutionProvider) == 0 || strcmp(target_ep->Type().c_str(), onnxruntime::kDmlExecutionProvider) == 0) { @@ -3136,7 +3127,6 @@ Status InferenceSession::Run(const RunOptions& run_options, // are needed before replaying the captured graph, here run N inference runs recursively until graph captured, // so that users just need one session run to capture the graph. // N is defined in min_num_runs_before_cuda_graph_capture_ for CUDA EP, - // N is defined in min_num_runs_before_hip_graph_capture_ for ROCM EP, // and the value could be different for other EP. if (retval.IsOK() && cached_execution_provider_for_graph_replay_.IsGraphCaptureEnabled() && cached_execution_provider_for_graph_replay_.AllowGraphCaptureOnRun(graph_annotation_id) && diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index c0e4d32ac0167..f3525d8de7b95 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -256,6 +256,7 @@ ORT_API_STATUS_IMPL(AddInitializer, _Inout_ OrtSessionOptions* options, _In_z_ c ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_CUDA, _In_ OrtSessionOptions* options, _In_ const OrtCUDAProviderOptions* cuda_options); + ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* rocm_options); ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_OpenVINO, diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index 48d52ae3cf428..e2ab0036c238f 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -628,6 +628,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_VitisAI, return CreateNotEnabledStatus("VitisAI"); } #endif + ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_ROCM, _In_ OrtSessionOptions* options, _In_ const OrtROCMProviderOptions* provider_options) { ORT_UNUSED_PARAMETER(options); diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index 4c3313046457c..88d7a11e4b0c4 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -537,16 +537,6 @@ def _create_inference_session(self, providers, provider_options, disabled_optimi self._fallback_providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] else: self._fallback_providers = ["CPUExecutionProvider"] - # MIGraphX can fall back to ROCM if it's explicitly assigned. All others fall back to CPU. - elif "MIGraphXExecutionProvider" in available_providers: - if providers and any( - provider == "ROCMExecutionProvider" - or (isinstance(provider, tuple) and provider[0] == "ROCMExecutionProvider") - for provider in providers - ): - self._fallback_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"] - else: - self._fallback_providers = ["CPUExecutionProvider"] else: self._fallback_providers = ["CPUExecutionProvider"] diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc index 1934e0eda7956..14330655e1ecc 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.cc @@ -154,14 +154,6 @@ OrtMemoryInfo GetMemoryInfoPerDeviceType(const OrtDevice& ort_device) { mem_info = GetCudaAllocator(ort_device.Id())->Info(); } #endif -#if USE_ROCM - else if (ort_device.Type() == OrtDevice::GPU) { - if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), ort_device.Id())) { - ORT_THROW("The provided device id doesn't match any available GPUs on the machine: ", ort_device.Id()); - } - mem_info = GetRocmAllocator(ort_device.Id())->Info(); - } -#endif #if USE_MIGRAPHX else if (ort_device.Type() == OrtDevice::GPU) { mem_info = GetMIGraphXAllocator(ort_device.Id())->Info(); @@ -440,55 +432,6 @@ AllocatorPtr GetCannAllocator(OrtDevice::DeviceId id) { #endif -#ifdef USE_ROCM -void CpuToRocmMemCpy(void* dst, const void* src, size_t num_bytes) { - GetProviderInfo_ROCM().rocmMemcpy_HostToDevice(dst, src, num_bytes); -} - -void RocmToCpuMemCpy(void* dst, const void* src, size_t num_bytes) { - GetProviderInfo_ROCM().rocmMemcpy_DeviceToHost(dst, src, num_bytes); -} - -const std::unordered_map* GetRocmToHostMemCpyFunction(const OrtDevice& device) { - static std::unordered_map map{ - {OrtDevice{OrtDevice::GPU, OrtDevice::MemType::DEFAULT, OrtDevice::VendorIds::AMD, device.Id()}, RocmToCpuMemCpy}, - }; - - return ↦ -} - -bool IsRocmDeviceIdValid(const onnxruntime::logging::Logger& logger, int id) { - int num_devices = GetProviderInfo_ROCM().hipGetDeviceCount(); - - if (0 == num_devices) { - LOGS(logger, WARNING) << "your system does not have a ROCM capable device."; - return false; - } - - if (id < 0 || id >= num_devices) { - LOGS(logger, WARNING) << "rocm_device=" << id << " is invalid, must choose device ID between 0 and " << num_devices - 1; - return false; - } - - return true; -} - -AllocatorPtr GetRocmAllocator(OrtDevice::DeviceId id) { - // Current approach is not thread-safe, but there are some bigger infra pieces to put together in order to make - // multi-threaded ROCM allocation work we need to maintain a per-thread ROCM allocator - - static auto* id_to_allocator_map = new std::unordered_map(); - - if (id_to_allocator_map->find(id) == id_to_allocator_map->end()) { - // TODO: Expose knobs so that users can set fields associated with OrtArenaCfg so that we can pass it to the following method - id_to_allocator_map->insert({id, GetProviderInfo_ROCM().CreateRocmAllocator(id, gpu_mem_limit, arena_extend_strategy, external_allocator_info, nullptr)}); - } - - return (*id_to_allocator_map)[id]; -} - -#endif - int OnnxRuntimeTensorToNumpyType(const DataTypeImpl* tensor_type) { static std::map type_map{ {DataTypeImpl::GetType(), NPY_BOOL}, diff --git a/onnxruntime/python/onnxruntime_pybind_mlvalue.h b/onnxruntime/python/onnxruntime_pybind_mlvalue.h index eba783d826212..377122a8bf73e 100644 --- a/onnxruntime/python/onnxruntime_pybind_mlvalue.h +++ b/onnxruntime/python/onnxruntime_pybind_mlvalue.h @@ -122,20 +122,6 @@ AllocatorPtr GetCannAllocator(OrtDevice::DeviceId id); #endif -#ifdef USE_ROCM - -bool IsRocmDeviceIdValid(const onnxruntime::logging::Logger& logger, int id); - -AllocatorPtr GetRocmAllocator(OrtDevice::DeviceId id); - -void CpuToRocmMemCpy(void* dst, const void* src, size_t num_bytes); - -void RocmToCpuMemCpy(void* dst, const void* src, size_t num_bytes); - -const std::unordered_map* GetRocmToHostMemCpyFunction(const OrtDevice&); - -#endif - void CreateGenericMLValue(const onnxruntime::InputDefList* input_def_list, const AllocatorPtr& alloc, const std::string& name_input, const pybind11::object& value, OrtValue* p_mlvalue, bool accept_only_numpy_array = false, bool use_numpy_data_memory = true, diff --git a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc index d74663ddb63d7..f996bf213b4a0 100644 --- a/onnxruntime/python/onnxruntime_pybind_ortvalue.cc +++ b/onnxruntime/python/onnxruntime_pybind_ortvalue.cc @@ -47,13 +47,7 @@ std::unique_ptr OrtValueFromShapeAndType(const std::vector& s "Please use the CUDA package of OnnxRuntime to use this feature."); #endif } else if (strcmp(GetDeviceName(device), HIP) == 0) { -#if USE_ROCM - if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - allocator = GetRocmAllocator(device.Id()); -#elif USE_MIGRAPHX +#if USE_MIGRAPHX allocator = GetMIGraphXAllocator(device.Id()); #else throw std::runtime_error( @@ -125,20 +119,6 @@ void addOrtValueMethods(pybind11::module& m) { true, false, CpuToCudaMemCpy); } else #endif -#ifdef USE_ROCM - if (device.Vendor() == OrtDevice::VendorIds::AMD) { - if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - // InputDeflist is null because OrtValue creation is not tied to a specific model - // Likewise, there is no need to specify the name (as the name was previously used to lookup the def list) - // TODO: Add check to ensure that string arrays are not passed - we currently don't support string tensors - // in ROCM - CreateGenericMLValue(nullptr, GetRocmAllocator(device.Id()), "", array_on_cpu, ml_value.get(), - true, false, CpuToRocmMemCpy); - } else -#endif #if USE_MIGRAPHX if (device.Vendor() == OrtDevice::VendorIds::AMD) { // InputDeflist is null because OrtValue creation is not tied to a specific model @@ -212,19 +192,6 @@ void addOrtValueMethods(pybind11::module& m) { CpuToCudaMemCpy); } else #endif -#if USE_ROCM - if (device.Vendor() == OrtDevice::VendorIds::AMD) { - if (!IsRocmDeviceIdValid(logging::LoggingManager::DefaultLogger(), device.Id())) { - throw std::runtime_error("The provided device id doesn't match any available GPUs on the machine."); - } - - onnxruntime::python::CopyDataToTensor( - py_values, - values_type, - *(ml_value->GetMutable()), - CpuToRocmMemCpy); - } else -#endif #if USE_MIGRAPHX if (device.Vendor() == OrtDevice::VendorIds::AMD) { onnxruntime::python::CopyDataToTensor( diff --git a/onnxruntime/python/onnxruntime_pybind_schema.cc b/onnxruntime/python/onnxruntime_pybind_schema.cc index cd1d2a8da10aa..8cb617fe5226c 100644 --- a/onnxruntime/python/onnxruntime_pybind_schema.cc +++ b/onnxruntime/python/onnxruntime_pybind_schema.cc @@ -29,12 +29,6 @@ void addGlobalSchemaFunctions(pybind11::module& m) { return CudaProviderFactoryCreator::Create(&provider_options); }(), #endif -#ifdef USE_ROCM - []() { - OrtROCMProviderOptions provider_options; - return onnxruntime::RocmProviderFactoryCreator::Create(&provider_options); - }(), -#endif #ifdef USE_DNNL onnxruntime::DnnlProviderFactoryCreator::Create(1), #endif diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index c548f3df4fb27..0bd0daf837645 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -509,27 +509,6 @@ const CANNExecutionProviderInfo GetCannExecutionProviderInfo(ProviderInfo_CANN* } #endif -#ifdef USE_ROCM -const ROCMExecutionProviderInfo GetRocmExecutionProviderInfo(ProviderInfo_ROCM* rocm_provider_info, - const ProviderOptionsMap& provider_options_map) { - ORT_ENFORCE(rocm_provider_info); - const auto it = provider_options_map.find(kRocmExecutionProvider); - ROCMExecutionProviderInfo info; - if (it != provider_options_map.end()) - rocm_provider_info->ROCMExecutionProviderInfo__FromProviderOptions(it->second, info); - else { - info.device_id = cuda_device_id; - info.gpu_mem_limit = gpu_mem_limit; - info.arena_extend_strategy = arena_extend_strategy; - info.miopen_conv_exhaustive_search = miopen_conv_exhaustive_search; - info.do_copy_in_default_stream = do_copy_in_default_stream; - info.external_allocator_info = external_allocator_info; - info.tunable_op = tunable_op; - } - return info; -} -#endif - #if defined(USE_TENSORRT) || defined(USE_TENSORRT_PROVIDER_INTERFACE) void RegisterTensorRTPluginsAsCustomOps(PySessionOptions& so, const ProviderOptions& options) { if (auto* tensorrt_provider_info = TryGetProviderInfo_TensorRT()) { @@ -1029,26 +1008,6 @@ static std::shared_ptr CreateExecutionProviderFactory "make sure they're in the PATH, and that your GPU is supported."; #endif // defined(USE_CUDA) #endif // defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) - } else if (type == kRocmExecutionProvider) { -#ifdef USE_ROCM - if (auto* rocm_provider_info = TryGetProviderInfo_ROCM()) { - const ROCMExecutionProviderInfo info = GetRocmExecutionProviderInfo(rocm_provider_info, - provider_options_map); - - // This variable is never initialized because the APIs by which is it should be initialized are deprecated, - // however they still exist and are in-use. Nevertheless, it is used to return ROCMAllocator, hence we must - // try to initialize it here if we can since FromProviderOptions might contain external ROCM allocator. - external_allocator_info = info.external_allocator_info; - return rocm_provider_info->CreateExecutionProviderFactory(info); - } else { - if (!Env::Default().GetEnvironmentVar("ROCM_PATH").empty()) { - ORT_THROW( - "ROCM_PATH is set but ROCM wasn't able to be loaded. Please install the correct version " - "of ROCM and MIOpen as mentioned in the GPU requirements page, make sure they're in the PATH, " - "and that your GPU is supported."); - } - } -#endif } else if (type == kDnnlExecutionProvider) { #ifdef USE_DNNL // Generate dnnl_options @@ -1475,7 +1434,7 @@ bool CheckIfTensor(const std::vector& def_list, } #if defined(USE_OPENVINO) || defined(USE_OPENVINO_PROVIDER_INTERFACE) || \ - defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) || defined(USE_ROCM) + defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) static void LogDeprecationWarning( const std::string& deprecated, const optional& alternative = nullopt) { LOGS_DEFAULT(WARNING) << "This is DEPRECATED and will be removed in the future: " << deprecated; @@ -1629,7 +1588,7 @@ void addGlobalMethods(py::module& m) { "Gets the dynamically selected OpenVINO device type for inference."); #endif -#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) /* * The following set_* methods are deprecated. * @@ -1639,40 +1598,30 @@ void addGlobalMethods(py::module& m) { */ // TODO remove deprecated global config m.def("set_cuda_device_id", [](const int id) { - LogDeprecationWarning("set_cuda_device_id", "CUDA/ROCM execution provider option \"device_id\""); + LogDeprecationWarning("set_cuda_device_id", "CUDA execution provider option \"device_id\""); cuda_device_id = static_cast(id); }); // TODO remove deprecated global config m.def("set_cudnn_conv_algo_search", [](const OrtCudnnConvAlgoSearch algo) { LogDeprecationWarning("set_cudnn_conv_algo_search", "CUDA execution provider option \"cudnn_conv_algo_search\""); -#ifdef USE_ROCM - ORT_UNUSED_PARAMETER(algo); - ORT_THROW("set_cudnn_conv_algo_search is not supported in ROCM"); -#else cudnn_conv_algo_search = algo; -#endif }); // TODO remove deprecated global config m.def("set_do_copy_in_default_stream", [](const bool use_single_stream) { LogDeprecationWarning( "set_do_copy_in_default_stream", "CUDA execution provider option \"do_copy_in_default_stream\""); -#ifdef USE_ROCM - ORT_UNUSED_PARAMETER(use_single_stream); - ORT_THROW("set_do_copy_in_default_stream is not supported in ROCM"); -#else do_copy_in_default_stream = use_single_stream; -#endif }); // TODO remove deprecated global config m.def("set_gpu_mem_limit", [](const int64_t limit) { LogDeprecationWarning( "set_gpu_mem_limit", - "CUDA execution provider option \"gpu_mem_limit\", ROCM execution provider option \"gpu_mem_limit\""); + "CUDA execution provider option \"gpu_mem_limit\""); gpu_mem_limit = gsl::narrow(limit); }); // TODO remove deprecated global config m.def("set_arena_extend_strategy", [](const onnxruntime::ArenaExtendStrategy strategy) { - LogDeprecationWarning("set_arena_extend_strategy", "CUDA/ROCM execution provider option \"arena_extend_strategy\""); + LogDeprecationWarning("set_arena_extend_strategy", "CUDA execution provider option \"arena_extend_strategy\""); arena_extend_strategy = strategy; }); #endif @@ -1825,7 +1774,7 @@ void addObjectMethods(py::module& m, ExecutionProviderRegistrationFn ep_registra } else if (type == OrtDevice::GPU) { #if USE_CUDA || USE_NV || USE_NV_PROVIDER_INTERFACE || USE_CUDA_PROVIDER_INTERFACE vendor = OrtDevice::VendorIds::NVIDIA; -#elif USE_ROCM || USE_MIGRAPHX +#elif USE_MIGRAPHX vendor = OrtDevice::VendorIds::AMD; #endif } else if (type == OrtDevice::NPU) { @@ -1892,7 +1841,7 @@ void addObjectMethods(py::module& m, ExecutionProviderRegistrationFn ep_registra py::class_ py_sync_stream(m, "OrtSyncStream", R"pbdoc(Represents a synchronization stream for model inference.)pbdoc"); - py_sync_stream.def("get_handle", [](OrtSyncStream* stream) -> uintptr_t { + py_sync_stream.def("get_handle", [](OrtSyncStream* stream) -> uintptr_t { Ort::UnownedSyncStream ort_stream(stream); return reinterpret_cast(ort_stream.GetHandle()); }, R"pbdoc(SyncStream handle that can be converted to a string and added to SessionOptions)pbdoc"); @@ -2006,7 +1955,7 @@ for model inference.)pbdoc"); .def_property_readonly("allocator_type", [](const OrtMemoryInfo* mem_info) -> OrtAllocatorType { return mem_info->alloc_type; }, R"pbdoc(Allocator type)pbdoc") .def_property_readonly("device_mem_type", [](const OrtMemoryInfo* mem_info) -> OrtDeviceMemoryType { auto mem_type = mem_info->device.MemType(); - return (mem_type == OrtDevice::MemType::DEFAULT) ? + return (mem_type == OrtDevice::MemType::DEFAULT) ? OrtDeviceMemoryType_DEFAULT: OrtDeviceMemoryType_HOST_ACCESSIBLE ; }, R"pbdoc(Device memory type (Device or Host accessible).)pbdoc") .def_property_readonly("device_vendor_id", [](const OrtMemoryInfo* mem_info) -> uint32_t { return mem_info->device.Vendor(); }); @@ -2748,7 +2697,7 @@ including arg name, arg type (contains both type and shape).)pbdoc") auto res = sess->GetSessionHandle()->GetModelMetadata(); OrtPybindThrowIfError(res.first); return *(res.second); }, py::return_value_policy::reference_internal) - .def_property_readonly("input_meminfos", [](const PyInferenceSession* sess) -> py::list { + .def_property_readonly("input_meminfos", [](const PyInferenceSession* sess) -> py::list { Ort::ConstSession session(reinterpret_cast(sess->GetSessionHandle())); auto inputs_mem_info = session.GetMemoryInfoForInputs(); py::list result; @@ -2757,7 +2706,7 @@ including arg name, arg type (contains both type and shape).)pbdoc") result.append(py::cast(p_info, py::return_value_policy::reference)); } return result; }) - .def_property_readonly("output_meminfos", [](const PyInferenceSession* sess) -> py::list { + .def_property_readonly("output_meminfos", [](const PyInferenceSession* sess) -> py::list { Ort::ConstSession session(reinterpret_cast(sess->GetSessionHandle())); auto outputs_mem_info = session.GetMemoryInfoForOutputs(); py::list result; diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.cc b/onnxruntime/python/onnxruntime_pybind_state_common.cc index cccdb9d23900a..0d00f3c4e6eb0 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.cc +++ b/onnxruntime/python/onnxruntime_pybind_state_common.cc @@ -31,17 +31,7 @@ onnxruntime::CUDAExecutionProviderExternalAllocatorInfo external_allocator_info{ onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; #endif -#ifdef USE_ROCM -// TODO remove deprecated global config -bool miopen_conv_exhaustive_search = false; -// TODO remove deprecated global config -bool do_copy_in_default_stream = true; -// TODO remove deprecated global config -onnxruntime::rocm::TunableOpInfo tunable_op{}; -onnxruntime::ROCMExecutionProviderExternalAllocatorInfo external_allocator_info{}; -#endif - -#if defined(USE_ROCM) || defined(USE_MIGRAPHX) +#if defined(USE_MIGRAPHX) // TODO remove deprecated global config onnxruntime::ArenaExtendStrategy arena_extend_strategy = onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo; #endif diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index b4a33e798f942..30ca76877dd0d 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -34,7 +34,7 @@ struct OrtStatus { #include "core/providers/tensorrt/tensorrt_provider_options.h" #include "core/providers/nv_tensorrt_rtx/nv_provider_options.h" -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #define BACKEND_PROC "GPU" #else #define BACKEND_PROC "CPU" @@ -122,10 +122,6 @@ struct OrtStatus { #include "core/providers/cuda/cuda_provider_factory.h" #include "core/providers/cuda/cuda_execution_provider_info.h" #endif -#ifdef USE_ROCM -#include "core/providers/rocm/rocm_provider_factory.h" -#include "core/providers/rocm/rocm_execution_provider_info.h" -#endif #if defined(USE_TENSORRT) || defined(USE_TENSORRT_PROVIDER_INTERFACE) #include "core/providers/tensorrt/tensorrt_provider_factory.h" #endif @@ -198,23 +194,7 @@ ProviderInfo_CANN& GetProviderInfo_CANN(); } // namespace onnxruntime #endif -#ifdef USE_ROCM -namespace onnxruntime { -ProviderInfo_ROCM* TryGetProviderInfo_ROCM(); -ProviderInfo_ROCM& GetProviderInfo_ROCM(); -namespace python { -// TODO remove deprecated global config -extern bool miopen_conv_exhaustive_search; -// TODO remove deprecated global config -extern bool do_copy_in_default_stream; -// TODO remove deprecated global config -extern onnxruntime::rocm::TunableOpInfo tunable_op; -extern onnxruntime::ROCMExecutionProviderExternalAllocatorInfo external_allocator_info; -} // namespace python -} // namespace onnxruntime -#endif - -#if defined(USE_ROCM) || defined(USE_MIGRAPHX) +#if defined(USE_MIGRAPHX) namespace onnxruntime { namespace python { extern onnxruntime::ArenaExtendStrategy arena_extend_strategy; diff --git a/onnxruntime/python/tools/microbench/benchmark.py b/onnxruntime/python/tools/microbench/benchmark.py index a5936afcfe13e..257548b612c73 100644 --- a/onnxruntime/python/tools/microbench/benchmark.py +++ b/onnxruntime/python/tools/microbench/benchmark.py @@ -32,12 +32,12 @@ def add_arguments(parser: ArgumentParser): "--provider", required=False, type=str, - choices=["cuda", "rocm", "cpu", None], + choices=["cuda", "cpu", None], default=None, help=( "Execution provider to use. By default, a " "provider is selected in the priority order " - "(cuda|rocm, cpu) depending on availability." + "(cuda, cpu) depending on availability." ), ) parser.add_argument( @@ -60,7 +60,6 @@ def add_arguments(parser: ArgumentParser): def provider_name(name): provider_map = { "cuda": "CUDAExecutionProvider", - "rocm": "ROCMExecutionProvider", "cpu": "CPUExecutionProvider", } return provider_map[name] @@ -69,8 +68,6 @@ def provider_name(name): def get_default_provider(): if "CUDAExecutionProvider" in ort.get_available_providers(): return "CUDAExecutionProvider" - if "ROCMExecutionProvider" in ort.get_available_providers(): - return "ROCMExecutionProvider" return "CPUExecutionProvider" @@ -85,7 +82,7 @@ def __init__(self, model, inputs, outputs, args): self.outputs = outputs def create_input_output_tensors(self): - on_gpu = self.provider == "CUDAExecutionProvider" or self.provider == "ROCMExecutionProvider" + on_gpu = self.provider == "CUDAExecutionProvider" device = "cuda" if on_gpu else "cpu" input_tensors = {name: torch.from_numpy(array).to(device) for name, array in self.inputs.items()} output_tensors = {name: torch.from_numpy(array).to(device) for name, array in self.outputs.items()} diff --git a/onnxruntime/python/tools/transformers/benchmark.py b/onnxruntime/python/tools/transformers/benchmark.py index 77a9e31b6208f..eb29080734b40 100644 --- a/onnxruntime/python/tools/transformers/benchmark.py +++ b/onnxruntime/python/tools/transformers/benchmark.py @@ -34,8 +34,6 @@ python benchmark.py -e torchscript -g -p "fp16" Run ONNXRuntime and TorchScript on CPU for all models with quantization: python benchmark.py -e torchscript onnxruntime -p "int8" -o - Run OnnxRuntime with the ROCM provider and graph optimization script: - python benchmark.py -g -m bert-base-cased --provider rocm --optimizer_info by_script --disable_embed_layer_norm Run OnnxRuntime with bfloat16 fastmath mode kernels on aarch64 platforms with bfloat16 support: python benchmark.py --enable_arm64_bfloat16_fastmath_mlas_gemm @@ -118,7 +116,6 @@ def run_onnxruntime( use_gpu and ("CUDAExecutionProvider" not in onnxruntime.get_available_providers()) and ("MIGraphXExecutionProvider" not in onnxruntime.get_available_providers()) - and ("ROCMExecutionProvider" not in onnxruntime.get_available_providers()) and ("DmlExecutionProvider" not in onnxruntime.get_available_providers()) ): logger.error( @@ -788,7 +785,7 @@ def main(): logger.error("fp16 is for GPU only") return - if args.precision == Precision.INT8 and args.use_gpu and args.provider not in ["migraphx", "rocm"]: + if args.precision == Precision.INT8 and args.use_gpu and args.provider not in ["migraphx"]: logger.error("int8 is for CPU only") return diff --git a/onnxruntime/python/tools/transformers/benchmark_helper.py b/onnxruntime/python/tools/transformers/benchmark_helper.py index a6716c8df3bc2..8055e5e4ae876 100644 --- a/onnxruntime/python/tools/transformers/benchmark_helper.py +++ b/onnxruntime/python/tools/transformers/benchmark_helper.py @@ -112,12 +112,9 @@ def create_onnxruntime_session( elif use_gpu: if provider == "dml": providers = ["DmlExecutionProvider", "CPUExecutionProvider"] - elif provider == "rocm": - providers = ["ROCMExecutionProvider", "CPUExecutionProvider"] elif provider == "migraphx": providers = [ "MIGraphXExecutionProvider", - "ROCMExecutionProvider", "CPUExecutionProvider", ] elif provider == "cuda" or provider is None: @@ -174,8 +171,8 @@ def prepare_environment(cache_dir, output_dir, use_gpu, provider=None): else: assert not set(onnxruntime.get_available_providers()).isdisjoint( - ["CUDAExecutionProvider", "ROCMExecutionProvider", "MIGraphXExecutionProvider"] - ), "Please install onnxruntime-gpu package, or install ROCm support, to test GPU inference." + ["CUDAExecutionProvider", "MIGraphXExecutionProvider"] + ), "Please install onnxruntime-gpu package, or install migraphx, to test GPU inference." logger.info(f"PyTorch Version:{torch.__version__}") logger.info(f"Transformers Version:{transformers.__version__}") diff --git a/onnxruntime/python/tools/transformers/bert_perf_test.py b/onnxruntime/python/tools/transformers/bert_perf_test.py index ebf44e49c89bb..9920a413b699e 100644 --- a/onnxruntime/python/tools/transformers/bert_perf_test.py +++ b/onnxruntime/python/tools/transformers/bert_perf_test.py @@ -80,12 +80,9 @@ def create_session( if use_gpu: if provider == "dml": execution_providers = ["DmlExecutionProvider", "CPUExecutionProvider"] - elif provider == "rocm": - execution_providers = ["ROCMExecutionProvider", "CPUExecutionProvider"] elif provider == "migraphx": execution_providers = [ "MIGraphXExecutionProvider", - "ROCMExecutionProvider", "CPUExecutionProvider", ] elif provider == "cuda": @@ -128,11 +125,8 @@ def create_session( if use_gpu: if provider == "dml": assert "DmlExecutionProvider" in session.get_providers() - elif provider == "rocm": - assert "ROCMExecutionProvider" in session.get_providers() elif provider == "migraphx": assert "MIGraphXExecutionProvider" in session.get_providers() - assert "ROCMExecutionProvider" in session.get_providers() elif provider == "cuda": assert "CUDAExecutionProvider" in session.get_providers() elif provider == "tensorrt": diff --git a/onnxruntime/python/tools/transformers/models/gpt2/convert_to_onnx.py b/onnxruntime/python/tools/transformers/models/gpt2/convert_to_onnx.py index f8b7dd80710ae..a4015f50fdc13 100644 --- a/onnxruntime/python/tools/transformers/models/gpt2/convert_to_onnx.py +++ b/onnxruntime/python/tools/transformers/models/gpt2/convert_to_onnx.py @@ -96,8 +96,8 @@ def parse_arguments(argv=None): "--provider", required=False, default=None, - choices=["dml", "rocm", "migraphx", "cuda", "tensorrt"], - help="use dml, rocm, cuda, tensorrt or migraphx for respective backend", + choices=["dml", "migraphx", "cuda", "tensorrt"], + help="use dml, cuda, tensorrt or migraphx for respective backend", ) parser.add_argument( diff --git a/onnxruntime/python/tools/transformers/models/llama/benchmark.py b/onnxruntime/python/tools/transformers/models/llama/benchmark.py index 61bfc950735af..dbe4799f20b9c 100644 --- a/onnxruntime/python/tools/transformers/models/llama/benchmark.py +++ b/onnxruntime/python/tools/transformers/models/llama/benchmark.py @@ -584,7 +584,7 @@ def get_args(rank=0): "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], ) parser.add_argument("-id", "--device-id", type=int, default=0) parser.add_argument("-w", "--warmup-runs", type=int, default=5) @@ -622,9 +622,6 @@ def get_args(rank=0): setattr(args, "execution_provider", f"{args.device.upper()}ExecutionProvider") # noqa: B010 if args.execution_provider == "CUDAExecutionProvider": args.execution_provider = (args.execution_provider, {"device_id": rank}) - elif args.execution_provider == "ROCMExecutionProvider": - args.execution_provider = (args.execution_provider, {"device_id": rank}) - args.device = "cuda" # Check that paths have been specified for any benchmarking with ORT if args.benchmark_type == "hf-ort": diff --git a/onnxruntime/python/tools/transformers/models/llama/benchmark_all.py b/onnxruntime/python/tools/transformers/models/llama/benchmark_all.py index 6447a7322b6ed..059a69e492554 100644 --- a/onnxruntime/python/tools/transformers/models/llama/benchmark_all.py +++ b/onnxruntime/python/tools/transformers/models/llama/benchmark_all.py @@ -109,7 +109,7 @@ def get_args(): "--device", type=str, required=True, - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], help="Device to benchmark models", ) diff --git a/onnxruntime/python/tools/transformers/models/llama/convert_to_onnx.py b/onnxruntime/python/tools/transformers/models/llama/convert_to_onnx.py index aa118da71525a..6411dca00b5de 100644 --- a/onnxruntime/python/tools/transformers/models/llama/convert_to_onnx.py +++ b/onnxruntime/python/tools/transformers/models/llama/convert_to_onnx.py @@ -631,7 +631,7 @@ def get_args(): "--execution_provider", required=False, default="cpu", - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], help="Execution provider to verify parity with", ) diff --git a/onnxruntime/python/tools/transformers/models/llama/llama_parity.py b/onnxruntime/python/tools/transformers/models/llama/llama_parity.py index 383101c8a3b72..f0aa07d3768b6 100644 --- a/onnxruntime/python/tools/transformers/models/llama/llama_parity.py +++ b/onnxruntime/python/tools/transformers/models/llama/llama_parity.py @@ -228,7 +228,7 @@ def get_args(argv: list[str]): "--execution_provider", required=False, default="cpu", - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], help="Execution provider to verify parity with", ) diff --git a/onnxruntime/python/tools/transformers/models/stable_diffusion/README.md b/onnxruntime/python/tools/transformers/models/stable_diffusion/README.md index 2506ffe8a3f50..12e6df53de577 100644 --- a/onnxruntime/python/tools/transformers/models/stable_diffusion/README.md +++ b/onnxruntime/python/tools/transformers/models/stable_diffusion/README.md @@ -158,28 +158,6 @@ pip install -r requirements/cuda12/requirements.txt ``` Finally, `pip install tensorrt` for Linux. For Windows, pip install the tensorrt wheel in the downloaded TensorRT zip file instead. -### Setup Environment (ROCm) - -It is recommended that the users run the model with ROCm 6.2 or newer and Python 3.10. You can follow the following to install ROCm 6.x: https://rocmdocs.amd.com/projects/install-on-linux/en/latest/install/quick-start.html -Note that Windows is not supported for ROCm at the moment. - -``` -pip install -r requirements/rocm/requirements.txt -``` - -AMD GPU version of PyTorch can be installed from [pytorch.org](https://pytorch.org/get-started/locally/) or [AMD Radeon repo](https://repo.radeon.com/rocm/manylinux/rocm-rel-6.2.3/). - -#### Install onnxruntime-rocm - -One option is to install prebuilt wheel from https://repo.radeon.com/rocm/manylinux like: -``` -wget https://repo.radeon.com/rocm/manylinux/rocm-rel-6.2.3/onnxruntime_rocm-1.18.0-cp310-cp310-linux_x86_64.whl -pip install onnxruntime_rocm-1.18.0-cp310-cp310-linux_x86_64.whl -``` - -If you want to use latest version of onnxruntime, you can build from source with Rocm 6.x following https://onnxruntime.ai/docs/build/eps.html#amd-rocm. -When the build is finished, you can install the wheel:`pip install build/Linux/Release/dist/*.whl`. - ### Export ONNX pipeline This step will export stable diffusion 1.5 to ONNX model in float32 using script from diffusers. @@ -258,16 +236,6 @@ python benchmark.py -b 1 -v 1.5 For the first command, '-p' specifies a directory of optimized ONNX pipeline as generated by optimize_pipeline.py. For the second command without '-p', we will use ORTPipelineForText2Image to export and optimize ONNX models for clip, unet and vae decoder. -On ROCm EP, use the following command instead: -``` -python benchmark.py -p ./sd1.5_onnx/fp16 -b 1 --tuning --provider rocm -v 1.5 -``` - -For ROCm EP, you can substitute `python benchmark.py` with `python -m onnxruntime.transformers.models.stable_diffusion.benchmark` since -the installed package is built from source. For CUDA, it is recommended to run `python benchmark.py` with the latest benchmark script. - -For ROCm EP, the `--tuning` is mandatory because we heavily rely on tuning to find the runable kernels for ORT `OpKernel`s. - The default parameters are stable diffusion version=1.5, height=512, width=512, steps=50, batch_count=5. Run `python benchmark.py --help` for more information. #### Stable Diffusion 3.x and Flux 1.0 @@ -303,12 +271,6 @@ pip install torch --upgrade --index-url https://download.pytorch.org/whl/cu117 python benchmark.py -e torch -b 1 --enable_torch_compile -v 1.5 ``` -For ROCm: -``` -pip install torch --upgrade --index-url https://download.pytorch.org/whl/rocm5.4.2 -python benchmark.py -e torch -b 1 --enable_torch_compile --provider rocm -v 1.5 -``` - Sometime, it complains ptxas not found when there are multiple CUDA versions installed. It can be fixed like `export TRITON_PTXAS_PATH=/usr/local/cuda-11.7/bin/ptxas` before running benchmark. Note that torch.compile is not supported in Windows: we encountered error `Windows not yet supported for torch.compile`. So it is excluded from RTX 3060 results of Windows. @@ -352,65 +314,6 @@ Here FMHA means Attention and MultiHeadAttention operators with Flash Attention The last two optimizations (Packed QKV and BiasAdd) are only available in nightly package. Compared to 1.14.1, nightly package has slight improvement in performance. -### Results on MI250X with 1 GCD - -With runtime tuning enabled, we get following performance number on one GCD of a MI250X GPU: - -| Optimizations | Average Latency (batch_size=1) | Memory in MB (batch_size=1) | Average Latency (batch_size=8) | Memory in MB (batch_size=8) | -| --------------------------------------------------------------------- | ------------------------------ | --------------------------- | ------------------------------ | --------------------------- | -| Raw FP32 models | 6.7 | 17,319 | 36.4 * | 33,787 | -| FP16 baseline | 4.1 | 8,945 | 24.0 * | 34,493 | -| FP16 baseline + FMHA | 2.6 | 4,886 | 15.0 | 10,146 | -| FP16 baseline + FMHA + NhwcConv | 2.4 | 4,952 | 14.8 | 9,632 | -| FP16 baseline + FMHA + NhwcConv + GroupNorm | 2.3 | 4,906 | 13.6 | 9,774 | -| FP16 baseline + FMHA + NhwcConv + GroupNorm + BiasSplitGelu | 2.2 | 4,910 | 12.5 | 9,646 | -| FP16 baseline + FMHA + NhwcConv + GroupNorm + BiasSplitGelu + BiasAdd | 2.2 | 4,910 | 12.5 | 9,778 | - -The entries marked with `*` produce suspicious output images. The might be numerical stability or correctness issue for the pipeline. The performance number is for reference only. - - -### Example Benchmark output - -Common settings for below test results: - -| model_name | disable_safety_checker | height | width | steps | batch_count | num_prompts | -| ------------------------------ | ---------------------- | ------ | ----- | ----- | ----------- | ----------- | -| runwayml/stable-diffusion-v1-5 | TRUE | 512 | 512 | 50 | 5 | 1 | - -#### Results of MI250X, 1 GCD (Ubuntu 20.04) - -| engine | version | provider | batch size | average latency | first run memory MB | second run memory MB | -| ----------- | ----------------------- | --------------------- | ---------- | --------------- | ------------------- | -------------------- | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 1 | 2.2 | 5,548 | 4,908 | -| torch | 1.12.1+rocm5.4 | - | 1 | 3.4 | 6,653 | 4,613 | -| torch | 2.0.0+rocm5.4.2 | default | 1 | 3.2 | 5,977 | 4,368 | -| torch | 2.0.0+rocm5.4.2 | compile | 1 | 3.0 | 5,869 | 4,266 | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 4 | 6.6 | 5,546 | 4,906 | -| torch | 1.12.1+rocm5.4 | - | 4 | 10.1 | 19,477 | 11,325 | -| torch | 2.0.0+rocm5.4.2 | default | 4 | 10.5 | 13,051 | 7,300 | -| torch | 2.0.0+rocm5.4.2 | compile | 4 | 9.2 | 12,879 | 7,190 | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 8 | 12.5 | 9,778 | 9,006 | -| torch | 1.12.1+rocm5.4 | - | 8 | 19.3 | 55,851 | 20,014 | -| torch | 2.0.0+rocm5.4.2 | default | 8 | 20.3 | 23,551 | 11,930 | -| torch | 2.0.0+rocm5.4.2 | compile | 8 | 17.8 | 23,303 | 11,800 | - -#### Results of MI100 (Ubuntu 20.04) - -| engine | version | provider | batch size | average latency | first run memory MB | second run memory MB | -| ----------- | ----------------------- | --------------------- | ---------- | --------------- | ------------------- | -------------------- | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 1 | 2.4 | 5,254 | 4,614 | -| torch | 1.12.1+rocm5.4 | - | 1 | 3.5 | 5,771 | 4,672 | -| torch | 2.0.0+rocm5.4.2 | default | 1 | 3.5 | 5,811 | 4,206 | -| torch | 2.0.0+rocm5.4.2 | compile | 1 | 3.1 | 5,774 | 4,168 | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 4 | 7.5 | 7,290 | 6,646 | -| torch | 1.12.1+rocm5.4 | - | 4 | 10.7 | 19,334 | 11,181 | -| torch | 2.0.0+rocm5.4.2 | default | 4 | 11.5 | 12,881 | 7,151 | -| torch | 2.0.0+rocm5.4.2 | compile | 4 | 10.0 | 12,740 | 7,073 | -| onnxruntime | 1.15.0+rocm5.4.2 | ROCM | 8 | 14.4 | 7,320 | 6,676 | -| torch | 1.12.1+rocm5.4 | - | 8 | 20.2 | 31,820 | 19,908 | -| torch | 2.0.0+rocm5.4.2 | default | 8 | 22.2 | 23,415 | 11,815 | -| torch | 2.0.0+rocm5.4.2 | compile | 8 | 19.3 | 23,154 | 11,667 | - ### Credits Some CUDA kernels (TensorRT Fused Attention, GroupNorm, SplitGelu and BiasAdd etc.) and demo diffusion were originally implemented in [TensorRT](https://github.com/nviDIA/TensorRT) by Nvidia. @@ -418,9 +321,6 @@ We use [Flash Attention v2](https://github.com/Dao-AILab/flash-attention) in Lin We use Memory efficient attention from [CUTLASS](https://github.com/NVIDIA/cutlass). The kernels were developed by Meta xFormers. The ONNX export script and pipeline for stable diffusion was developed by Huggingface [diffusers](https://github.com/huggingface/diffusers) library. -Most ROCm kernel optimizations are from [composable kernel](https://github.com/ROCmSoftwarePlatform/composable_kernel). -Some kernels are enabled by MIOpen. We hereby thank for the AMD developers' collaboration. - ### Future Works * Update demo to support inpainting. * Support flash attention in Windows. diff --git a/onnxruntime/python/tools/transformers/models/stable_diffusion/benchmark.py b/onnxruntime/python/tools/transformers/models/stable_diffusion/benchmark.py index b4d2977050bd4..ed2e346972a6c 100755 --- a/onnxruntime/python/tools/transformers/models/stable_diffusion/benchmark.py +++ b/onnxruntime/python/tools/transformers/models/stable_diffusion/benchmark.py @@ -31,7 +31,6 @@ PROVIDERS = { "cuda": "CUDAExecutionProvider", - "rocm": "ROCMExecutionProvider", "migraphx": "MIGraphXExecutionProvider", "tensorrt": "TensorrtExecutionProvider", } @@ -328,7 +327,7 @@ def run_ort( skip_warmup: bool = False, ): provider_and_options = provider - if tuning and provider in ["CUDAExecutionProvider", "ROCMExecutionProvider"]: + if tuning and provider in ["CUDAExecutionProvider"]: provider_and_options = (provider, {"tunable_op_enable": 1, "tunable_op_tuning_enable": 1}) load_start = time.time() @@ -1150,8 +1149,7 @@ def parse_arguments(): "-t", "--tuning", action="store_true", - help="Enable TunableOp and tuning. " - "This will incur longer warmup latency, and is mandatory for some operators of ROCm EP.", + help="Enable TunableOp and tuning. This will incur longer warmup latency.", ) parser.add_argument( @@ -1336,7 +1334,7 @@ def main(): coloredlogs.install(fmt="%(funcName)20s: %(message)s") - memory_monitor_type = "rocm" if args.provider == "rocm" else "cuda" + memory_monitor_type = "cuda" start_memory = measure_gpu_memory(memory_monitor_type, None) print("GPU memory used before loading models:", start_memory) diff --git a/onnxruntime/python/tools/transformers/models/stable_diffusion/requirements/rocm/requirements.txt b/onnxruntime/python/tools/transformers/models/stable_diffusion/requirements/rocm/requirements.txt deleted file mode 100644 index 21b100fb61f17..0000000000000 --- a/onnxruntime/python/tools/transformers/models/stable_diffusion/requirements/rocm/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ --r ../requirements.txt -# Install onnxruntime-rocm that is built from source (https://onnxruntime.ai/docs/build/eps.html#amd-rocm) diff --git a/onnxruntime/python/tools/transformers/models/whisper/benchmark.py b/onnxruntime/python/tools/transformers/models/whisper/benchmark.py index 88fdad01baf92..04b62f4b2da99 100644 --- a/onnxruntime/python/tools/transformers/models/whisper/benchmark.py +++ b/onnxruntime/python/tools/transformers/models/whisper/benchmark.py @@ -130,9 +130,6 @@ def get_model(args: argparse.Namespace): if args.verbose: sess_options.log_verbosity_level = 1 sess_options.log_severity_level = 1 - if args.tune: - ort.set_default_logger_severity(0) - ort.set_default_logger_verbosity(0) else: raise Exception(f"Cannot recognize {args.benchmark_type}") @@ -338,9 +335,6 @@ def prepare_ort_inputs(inputs, warmup=False): logger.error(f"The following model inputs are missing: {missing_inputs}") raise Exception("There are missing inputs to the model. Please add them and try again.") - if warmup and args.tune: - inputs["min_length"] = inputs["max_length"] - # Remove unnecessary inputs from model inputs unnecessary_inputs = user_inputs - model_inputs if len(unnecessary_inputs): @@ -392,9 +386,6 @@ def handle_output(output): # ORT evaluation logger.info("\nEvaluating ONNX Runtime...") ort_evaluate_inputs = ort_inputs - if args.tune: - ort_warmup_inputs = prepare_ort_inputs(inputs, warmup=True) - ort_evaluate_inputs = (ort_warmup_inputs, ort_inputs) time_fn(args, generate_fn, ort_evaluate_inputs) ort_outputs = generate_fn(ort_inputs) @@ -479,7 +470,7 @@ def parse_args(): "--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu", - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], ) parser.add_argument("-id", "--device-id", type=int, default=0) parser.add_argument("-w", "--warmup-runs", type=int, default=5) @@ -527,12 +518,6 @@ def parse_args(): parser.add_argument("--pt-num-rows", type=int, default=1000, help="Number of rows for PyTorch profiler to display") parser.add_argument("--verbose", default=False, action="store_true") parser.add_argument("--log-folder", type=str, default=os.path.join("."), help="Folder to cache log files") - parser.add_argument( - "--tune", - default=False, - action="store_true", - help="Only used by ROCm EP, enable TunableOp tuning to select fastest kernel", - ) args = parser.parse_args() @@ -546,16 +531,6 @@ def parse_args(): args.execution_provider = f"{args.device.upper()}ExecutionProvider" if args.execution_provider == "CUDAExecutionProvider": args.execution_provider = (args.execution_provider, {"device_id": args.device_id}) - elif args.execution_provider == "ROCMExecutionProvider": - args.execution_provider = ( - args.execution_provider, - { - "device_id": args.device_id, - "tunable_op_enable": 1, - "tunable_op_tuning_enable": 1 if args.tune else 0, - }, - ) - args.device = "cuda" # Check that model paths have been specified for any benchmarking with ORT if args.benchmark_type == "hf-ort": diff --git a/onnxruntime/python/tools/transformers/models/whisper/benchmark_all.py b/onnxruntime/python/tools/transformers/models/whisper/benchmark_all.py index 95d4b60fead99..a5679fbc2c40e 100644 --- a/onnxruntime/python/tools/transformers/models/whisper/benchmark_all.py +++ b/onnxruntime/python/tools/transformers/models/whisper/benchmark_all.py @@ -105,7 +105,7 @@ def get_args(): "--device", type=str, required=True, - choices=["cpu", "cuda", "rocm"], + choices=["cpu", "cuda"], help="Device to benchmark models", ) diff --git a/onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py b/onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py index 38fbd73e9c119..79b508047da55 100644 --- a/onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py +++ b/onnxruntime/python/tools/transformers/models/whisper/convert_to_onnx.py @@ -26,7 +26,6 @@ PROVIDERS = { "cpu": "CPUExecutionProvider", "cuda": "CUDAExecutionProvider", - "rocm": "ROCMExecutionProvider", } diff --git a/onnxruntime/python/tools/transformers/models/whisper/whisper_helper.py b/onnxruntime/python/tools/transformers/models/whisper/whisper_helper.py index 85b3632c516ca..72c51386dfe9e 100644 --- a/onnxruntime/python/tools/transformers/models/whisper/whisper_helper.py +++ b/onnxruntime/python/tools/transformers/models/whisper/whisper_helper.py @@ -767,7 +767,7 @@ def optimize_onnx( optimization_options = FusionOptions("bart") optimization_options.use_multi_head_attention = True - optimization_options.disable_multi_head_attention_bias = provider == "rocm" + optimization_options.disable_multi_head_attention_bias = False m = optimize_model( onnx_model_path, diff --git a/onnxruntime/python/tools/transformers/optimizer.py b/onnxruntime/python/tools/transformers/optimizer.py index 32a25ef1420ba..f4e8bcbe9103e 100644 --- a/onnxruntime/python/tools/transformers/optimizer.py +++ b/onnxruntime/python/tools/transformers/optimizer.py @@ -111,7 +111,7 @@ def optimize_by_onnxruntime( use_gpu and provider is None and set(onnxruntime.get_available_providers()).isdisjoint( - ["CUDAExecutionProvider", "ROCMExecutionProvider", "MIGraphXExecutionProvider"] + ["CUDAExecutionProvider", "MIGraphXExecutionProvider"] ) ): logger.error("There is no gpu for onnxruntime to do optimization.") @@ -172,10 +172,8 @@ def optimize_by_onnxruntime( elif provider is not None: if provider == "dml": providers = ["DmlExecutionProvider"] - elif provider == "rocm": - providers = ["ROCMExecutionProvider"] elif provider == "migraphx": - providers = ["MIGraphXExecutionProvider", "ROCMExecutionProvider"] + providers = ["MIGraphXExecutionProvider"] elif provider == "cuda": providers = ["CUDAExecutionProvider"] elif provider == "tensorrt": @@ -189,7 +187,6 @@ def optimize_by_onnxruntime( if torch_version.hip: providers.append("MIGraphXExecutionProvider") - providers.append("ROCMExecutionProvider") else: providers.append("CUDAExecutionProvider") diff --git a/onnxruntime/test/contrib_ops/attention_op_test.cc b/onnxruntime/test/contrib_ops/attention_op_test.cc index 99fd3c18e94ef..411629535254d 100644 --- a/onnxruntime/test/contrib_ops/attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/attention_op_test.cc @@ -57,7 +57,6 @@ static void RunAttentionTest( int max_sequence_length = 0, const bool disable_cpu = false, const bool disable_cuda = false, - const bool disable_rocm = false, const bool disable_dml = false, const bool disable_webgpu = false, std::vector qkv_sizes = {}, @@ -72,22 +71,21 @@ static void RunAttentionTest( int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture) && !is_weights_constant && !disable_cuda; - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()) && !is_weights_constant && !disable_rocm; bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get()) && !use_float16 && !disable_cpu; bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()) && !disable_dml; bool enable_webgpu = (nullptr != DefaultWebGpuExecutionProvider().get()) && !disable_webgpu; int head_size = hidden_size / number_of_heads; - if (enable_cpu || enable_cuda || enable_rocm || enable_dml || enable_webgpu) { + if (enable_cpu || enable_cuda || enable_dml || enable_webgpu) { OpTester tester("Attention", 1, onnxruntime::kMSDomain); tester.AddAttribute("num_heads", static_cast(number_of_heads)); tester.AddAttribute("unidirectional", static_cast(is_unidirectional ? 1 : 0)); tester.AddAttribute("past_present_share_buffer", static_cast(past_present_share_buffer ? 1 : 0)); tester.AddAttribute("mask_filter_value", static_cast(-10000.0f)); - if (use_scale && !enable_rocm) { + if (use_scale) { tester.AddAttribute("scale", static_cast(1.f / sqrt(head_size))); } - if (do_neox_rotary && !enable_rocm) { + if (do_neox_rotary) { tester.AddAttribute("do_rotary", static_cast(do_neox_rotary ? 1 : 0)); } @@ -241,18 +239,6 @@ static void RunAttentionTest( tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } - if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider()); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } - - if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/true)); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } - if (enable_cpu) { std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -297,7 +283,6 @@ static void RunAttentionTest( int max_sequence_length = 0, const bool disable_cpu = false, const bool disable_cuda = false, - const bool disable_rocm = false, const bool disable_dml = false, const bool disable_webgpu = false, const std::vector qkv_sizes = {}, @@ -310,13 +295,13 @@ static void RunAttentionTest( batch_size, sequence_length, hidden_size, number_of_heads, use_float16, is_unidirectional, use_past_state, past_sequence_length, past_data, present_data, mask_type, input_hidden_size, max_sequence_length, - disable_cpu, disable_cuda, disable_rocm, disable_dml, disable_webgpu, qkv_sizes, attention_bias_data, + disable_cpu, disable_cuda, disable_dml, disable_webgpu, qkv_sizes, attention_bias_data, kv_sequence_length, past_present_share_buffer, use_scale, do_neox_rotary); RunAttentionTest(input_data, weights_data, true, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, use_float16, is_unidirectional, use_past_state, past_sequence_length, past_data, present_data, mask_type, input_hidden_size, max_sequence_length, - disable_cpu, disable_cuda, disable_rocm, disable_dml, disable_webgpu, qkv_sizes, attention_bias_data, + disable_cpu, disable_cuda, disable_dml, disable_webgpu, qkv_sizes, attention_bias_data, kv_sequence_length, past_present_share_buffer, use_scale, do_neox_rotary); } @@ -383,11 +368,10 @@ TEST(ContribOpAttentionTest, AttentionBatch1WithQKVAttr1) { 3.1967618465423584f, 0.51903456449508667f, 0.63051539659500122f, 2.9394614696502686f, 0.65332180261611938f, 1.000949501991272f, 0.74175024032592773f, 2.8231701850891113f}; - constexpr bool disable_rocm = true; RunAttentionTest(input_data, weight_data, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, false, false, false, 0, nullptr, nullptr, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, - 0, false, false, disable_rocm, false, false, qkv_sizes); + 0, false, false, false, false, qkv_sizes); } TEST(ContribOpAttentionTest, AttentionBatch1WithQKVAttr2) { @@ -421,11 +405,10 @@ TEST(ContribOpAttentionTest, AttentionBatch1WithQKVAttr2) { std::vector output_data = { 0.64932525157928467f, 0.79390722513198853f, 0.64932847023010254f, 0.79375863075256348f}; - constexpr bool disable_rocm = true; RunAttentionTest(input_data, weight_data, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, false, false, false, 0, nullptr, nullptr, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, - 0, false, false, disable_rocm, false, false, qkv_sizes); + 0, false, false, false, false, qkv_sizes); } TEST(ContribOpAttentionTest, AttentionBatch1AttentionBias) { @@ -461,12 +444,11 @@ TEST(ContribOpAttentionTest, AttentionBatch1AttentionBias) { constexpr bool disable_cpu = false; constexpr bool disable_cuda = false; - constexpr bool disable_rocm = false; constexpr bool disable_dml = false; RunAttentionTest(input_data, weight_data, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, false, false, false, 0, nullptr, nullptr, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, - 0, disable_cpu, disable_cuda, disable_rocm, disable_dml, false, qkv_sizes, attention_bias); + 0, disable_cpu, disable_cuda, disable_dml, false, qkv_sizes, attention_bias); } TEST(ContribOpAttentionTest, AttentionBatch2AttentionBias) { @@ -507,12 +489,11 @@ TEST(ContribOpAttentionTest, AttentionBatch2AttentionBias) { constexpr bool disable_cpu = false; constexpr bool disable_cuda = false; - constexpr bool disable_rocm = false; constexpr bool disable_dml = false; RunAttentionTest(input_data, weight_data, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, false, false, false, 0, nullptr, nullptr, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, - 0, disable_cpu, disable_cuda, disable_rocm, disable_dml, false, qkv_sizes, attention_bias); + 0, disable_cpu, disable_cuda, disable_dml, false, qkv_sizes, attention_bias); } TEST(ContribOpAttentionTest, AttentionBatch1_Float16) { @@ -859,7 +840,7 @@ void RawAttentionEmptyPastState(bool past_present_share_buffer) { RunAttentionTest(input_data, weight_data, bias_data, mask_index_data, output_data, batch_size, sequence_length, hidden_size, number_of_heads, false, is_unidirectional, use_past_state, past_sequence_length, &past_data, &present_data, - AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, sequence_length, true, false, true, disable_dml, true, + AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, sequence_length, true, false, disable_dml, true, {}, {}, 0, true); } } @@ -1042,7 +1023,7 @@ void RawAttentionPastStateBatch1(bool past_present_share_buffer) { batch_size, sequence_length, hidden_size, number_of_heads, false, is_unidirectional, use_past_state, past_sequence_length, &past_data, &present_data, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, past_sequence_length + sequence_length + 4, - true, false, true, disable_dml, true, {}, {}, 0, true); + true, false, disable_dml, true, {}, {}, 0, true); } } @@ -1175,7 +1156,7 @@ void RawAttentionPastStateBatch2(bool past_present_share_buffer) { batch_size, sequence_length, hidden_size, number_of_heads, false, is_unidirectional, use_past_state, past_sequence_length, &past_data, &present_data, AttentionMaskType::MASK_1D_KEY_SEQ_LEN, 0, past_sequence_length + sequence_length, - true, false, true, disable_dml, true, {}, {}, 0, true); + true, false, disable_dml, true, {}, {}, 0, true); } } @@ -1300,7 +1281,7 @@ void RawAttentionPastStateBatch2WithPadding(bool past_present_share_buffer) { use_past_state, past_sequence_length, &past_data, &present_data, AttentionMaskType::MASK_1D_END_START, 0, past_sequence_length + sequence_length + 4, - true, false, true, disable_dml, true, {}, {}, 0, true); + true, false, disable_dml, true, {}, {}, 0, true); } } @@ -1687,7 +1668,7 @@ TEST(ContribOpAttentionTest, AttentionWithNormFactor) { batch_size, sequence_length, hidden_size, number_of_heads, use_float16, is_unidirectional, use_past_state, past_sequence_length, past_data, present_data, AttentionMaskType::MASK_2D_KEY_PADDING, 0 /*input_hidden_size*/, 0 /*max_sequence_length*/, - false /*disable_cpu*/, false /*disable_cuda*/, true /*disable_rocm*/, false /*disable_dml*/, + false /*disable_cpu*/, false /*disable_cuda*/, false /*disable_dml*/, false /*disable_webgpu*/, {} /*qkv_sizes*/, {} /*attention_bias_data*/, 0 /*kv_sequence_length*/, false /*past_present_share_buffer*/, true /*use_scale*/); } @@ -1721,7 +1702,7 @@ TEST(ContribOpAttentionTest, AttentionWithNeoXRotaryEmbedding) { batch_size, sequence_length, hidden_size, number_of_heads, use_float16, is_unidirectional, use_past_state, past_sequence_length, past_data, present_data, AttentionMaskType::MASK_2D_KEY_PADDING, 0 /*input_hidden_size*/, 0 /*max_sequence_length*/, - true /*disable_cpu*/, false /*disable_cuda*/, true /*disable_rocm*/, disable_dml, + true /*disable_cpu*/, false /*disable_cuda*/, disable_dml, true /*disable_webgpu*/, {} /*qkv_sizes*/, {} /*attention_bias_data*/, 0 /*kv_sequence_length*/, false /*past_present_share_buffer*/, true /*use_scale*/, true /*use_neox_rotary_embedding*/); } @@ -1983,7 +1964,7 @@ TEST(ContribOpAttentionTest, Attention4DMask) { batch_size, sequence_length, hidden_size, number_of_heads, use_float16, is_unidirectional, use_past_state, past_sequence_length, past_data, present_data, AttentionMaskType::MASK_4D_MEGATRON, input_hidden_size, max_sequence_length, - disable_cpu, /* disable_cuda */ false, /* disable_rocm */ false, /* disable_dml */ false, /* disable_webgpu */ true); + disable_cpu, /* disable_cuda */ false, /* disable_dml */ false, /* disable_webgpu */ true); } TEST(ContribOpAttentionTest, AttentionMaskIndexOutOfRange) { @@ -2137,10 +2118,9 @@ static void RunModelWithRandomInput( float gpu_threshold = is_float16 ? 0.5f : 0.005f; constexpr float cpu_threshold = 0.002f; bool enable_cuda = HasCudaEnvironment(is_float16 ? 530 : 0); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get() && !is_float16); bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()); - if (enable_cuda || enable_rocm || enable_dml) { + if (enable_cuda || enable_dml) { OpTester test("Attention", 1, onnxruntime::kMSDomain); test.AddAttribute("num_heads", num_heads); if (is_float16) { @@ -2162,8 +2142,6 @@ static void RunModelWithRandomInput( execution_providers.push_back(DefaultCudaExecutionProvider()); } else if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); - } else { - execution_providers.push_back(DefaultRocmExecutionProvider()); } test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/onnxruntime/test/contrib_ops/beam_search_test.cc b/onnxruntime/test/contrib_ops/beam_search_test.cc index 20eea2138340f..f875c710046ac 100644 --- a/onnxruntime/test/contrib_ops/beam_search_test.cc +++ b/onnxruntime/test/contrib_ops/beam_search_test.cc @@ -83,11 +83,6 @@ void RunGptBeamSearchFp32() { session_options.AppendExecutionProvider_CUDA_V2(cuda_options); #endif -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - session_options.AppendExecutionProvider_ROCM(rocm_options); -#endif - // The ONNX model is generated like the following: // python convert_generation.py --model_type gpt2 -m hf-internal-testing/tiny-random-gpt2 // --output tiny_gpt2_beamsearch_fp16.onnx --use_gpu --max_length 20 @@ -177,8 +172,7 @@ TEST(BeamSearchTest, GptBeamSearchFp16) { constexpr int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); - if (enable_cuda || enable_rocm) { + if (enable_cuda) { Ort::SessionOptions session_options; #ifdef USE_CUDA OrtCUDAProviderOptionsV2 cuda_options; @@ -186,11 +180,6 @@ TEST(BeamSearchTest, GptBeamSearchFp16) { session_options.AppendExecutionProvider_CUDA_V2(cuda_options); #endif -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - session_options.AppendExecutionProvider_ROCM(rocm_options); -#endif - // The ONNX model is generated like the following: // python convert_generation.py --model_type gpt2 -m hf-internal-testing/tiny-random-gpt2 // --output tiny_gpt2_beamsearch_fp16.onnx -p fp16 --use_gpu --max_length 20 @@ -272,8 +261,7 @@ TEST(BeamSearchTest, GptBeamSearchWithInitDecoderFp16) { constexpr int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); - if (enable_cuda || enable_rocm) { + if (enable_cuda) { Ort::SessionOptions session_options; #ifdef USE_CUDA OrtCUDAProviderOptionsV2 cuda_options; @@ -281,11 +269,6 @@ TEST(BeamSearchTest, GptBeamSearchWithInitDecoderFp16) { session_options.AppendExecutionProvider_CUDA_V2(cuda_options); #endif -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - session_options.AppendExecutionProvider_ROCM(rocm_options); -#endif - // The ONNX model is generated like the following: // python convert_generation.py --model_type gpt2 -m hf-internal-testing/tiny-random-gpt2 // --output tiny_gpt2_beamsearch_with_init_decoder_fp16.onnx -p fp16 --use_gpu --max_length 20 @@ -366,8 +349,7 @@ TEST(BeamSearchTest, GptBeamSearchFp16_VocabPadded) { constexpr int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); - if (enable_cuda || enable_rocm) { + if (enable_cuda) { Ort::SessionOptions session_options; #ifdef USE_CUDA OrtCUDAProviderOptionsV2 cuda_options; @@ -375,11 +357,6 @@ TEST(BeamSearchTest, GptBeamSearchFp16_VocabPadded) { session_options.AppendExecutionProvider_CUDA_V2(cuda_options); #endif -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - session_options.AppendExecutionProvider_ROCM(rocm_options); -#endif - // The following model was obtained by padding the vocabulary size in testdata/transformers/tiny_gpt2_beamsearch_fp16.onnx // from 1000 to 1600 (just for illustrative and testing purposes) to see if the beam search implementation can handle // such a scenario diff --git a/onnxruntime/test/contrib_ops/bias_add_op_test.cc b/onnxruntime/test/contrib_ops/bias_add_op_test.cc index 6fd091ef66110..1ec51631ca9ca 100644 --- a/onnxruntime/test/contrib_ops/bias_add_op_test.cc +++ b/onnxruntime/test/contrib_ops/bias_add_op_test.cc @@ -13,7 +13,7 @@ using namespace onnxruntime::test; namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_DML) static std::vector GetExpectedResult(const std::vector& input_data, const std::vector& bias_data, const std::vector& skip_data) { @@ -38,10 +38,9 @@ static void RunSkipBiasGpuTest(const std::vector& input_data, bool use_float16 = false) { int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()); - if (!enable_cuda && !enable_rocm && !enable_dml) { + if (!enable_cuda && !enable_dml) { return; } @@ -63,9 +62,7 @@ static void RunSkipBiasGpuTest(const std::vector& input_data, if (enable_cuda) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); } diff --git a/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc b/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc index 027d4b3fff1b0..4852269a5b6b6 100644 --- a/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc +++ b/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -// BiasDropout kernel is only implemented for CUDA/ROCM -#if (defined(USE_CUDA) && !defined(USE_CUDA_MINIMAL)) || defined(USE_ROCM) +// BiasDropout kernel is only implemented for CUDA +#if (defined(USE_CUDA) && !defined(USE_CUDA_MINIMAL)) #ifdef _MSC_VER #pragma warning(disable : 4389) @@ -17,23 +17,14 @@ #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" #include "test/util/include/default_providers.h" -#ifdef USE_ROCM -#include "core/providers/rocm/shared_inc/rocm_utils.h" -#else #include "core/providers/cuda/shared_inc/cuda_utils.h" -#endif namespace onnxruntime { namespace contrib { namespace test { -#ifdef USE_ROCM -using onnxruntime::rocm::BitmaskElementType; -using onnxruntime::rocm::kNumBitsPerBitmaskElement; -#else using onnxruntime::cuda::BitmaskElementType; using onnxruntime::cuda::kNumBitsPerBitmaskElement; -#endif using namespace onnxruntime::test; enum TrainingMode { TrainingFalse, @@ -182,8 +173,6 @@ void RunBiasDropoutTest(const bool use_mask, const std::vector& input_s std::vector> t_eps; #ifdef USE_CUDA t_eps.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - t_eps.emplace_back(DefaultRocmExecutionProvider()); #endif t.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &t_eps); @@ -204,8 +193,6 @@ void RunBiasDropoutTest(const bool use_mask, const std::vector& input_s std::vector> t_bitmask_eps; #ifdef USE_CUDA t_bitmask_eps.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - t_bitmask_eps.emplace_back(DefaultRocmExecutionProvider()); #endif t_bitmask.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &t_bitmask_eps); } diff --git a/onnxruntime/test/contrib_ops/bias_softmax_op_test.cc b/onnxruntime/test/contrib_ops/bias_softmax_op_test.cc index bada23723e9f4..54c7cc16bc1a8 100644 --- a/onnxruntime/test/contrib_ops/bias_softmax_op_test.cc +++ b/onnxruntime/test/contrib_ops/bias_softmax_op_test.cc @@ -13,12 +13,6 @@ namespace onnxruntime { namespace test { -#if USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; -#else -constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#endif - // followed example of fastgelu_op_test.cc // in retrospect would have been better to compare BiasSoftmax to Add + Softmax graph @@ -134,7 +128,7 @@ class BiasSoftmaxTester { void RunComparison() { // BiasSoftmax only implemented for cuda architecture int min_cuda_architecture = use_float16_ ? 530 : 0; - if (HasCudaEnvironment(min_cuda_architecture) || kGpuExecutionProvider == kRocmExecutionProvider) { + if (HasCudaEnvironment(min_cuda_architecture)) { OpTester tester("BiasSoftmax", 1, onnxruntime::kMSDomain); tester.AddAttribute("axis", axis_); tester.AddAttribute("is_inner_broadcast", is_inner_broadcast_); @@ -152,8 +146,6 @@ class BiasSoftmaxTester { std::vector> ep; #ifdef USE_CUDA ep.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - ep.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &ep); diff --git a/onnxruntime/test/contrib_ops/bias_split_gelu_op_test.cc b/onnxruntime/test/contrib_ops/bias_split_gelu_op_test.cc index a979717d23573..42db1c1201b63 100644 --- a/onnxruntime/test/contrib_ops/bias_split_gelu_op_test.cc +++ b/onnxruntime/test/contrib_ops/bias_split_gelu_op_test.cc @@ -74,7 +74,7 @@ std::vector GetExpectedResult(const std::vector& input_data, } } // namespace bias_split_gelu_test -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_DML) static void RunBiasSplitGeluGpuTest(const std::vector& input_data, const std::vector& bias_data, @@ -85,10 +85,9 @@ static void RunBiasSplitGeluGpuTest(const std::vector& input_data, bool use_float16 = false) { int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()); - if (!enable_cuda && !enable_rocm && !enable_dml) { + if (!enable_cuda && !enable_dml) { return; } @@ -108,9 +107,7 @@ static void RunBiasSplitGeluGpuTest(const std::vector& input_data, if (enable_cuda) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); } diff --git a/onnxruntime/test/contrib_ops/bitmask_dropout_op_test.cc b/onnxruntime/test/contrib_ops/bitmask_dropout_op_test.cc index 7ca4e1004066c..926c45cadcc1b 100644 --- a/onnxruntime/test/contrib_ops/bitmask_dropout_op_test.cc +++ b/onnxruntime/test/contrib_ops/bitmask_dropout_op_test.cc @@ -1,29 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" #include "test/common/tensor_op_test_utils.h" #include "test/util/include/default_providers.h" -#ifdef USE_ROCM -#include "core/providers/rocm/shared_inc/rocm_utils.h" -#else #include "core/providers/cuda/shared_inc/cuda_utils.h" -#endif namespace onnxruntime { namespace contrib { namespace test { -#ifdef USE_ROCM -using onnxruntime::rocm::BitmaskElementType; -using onnxruntime::rocm::kNumBitsPerBitmaskElement; -#else using onnxruntime::cuda::BitmaskElementType; using onnxruntime::cuda::kNumBitsPerBitmaskElement; -#endif using namespace onnxruntime::test; namespace { @@ -62,8 +53,6 @@ void RunTestForInference(const std::vector& input_dims, bool has_ratio std::vector> test_eps; #ifdef USE_CUDA test_eps.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - test_eps.emplace_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &test_eps); } @@ -123,8 +112,6 @@ void RunTestForTraining(const std::vector& input_dims) { std::vector> dropout_eps; #ifdef USE_CUDA dropout_eps.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - dropout_eps.emplace_back(DefaultRocmExecutionProvider()); #endif dropout.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &dropout_eps); @@ -146,8 +133,6 @@ void RunTestForTraining(const std::vector& input_dims) { std::vector> bitmask_dropout_eps; #ifdef USE_CUDA bitmask_dropout_eps.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - bitmask_dropout_eps.emplace_back(DefaultRocmExecutionProvider()); #endif bitmask_dropout.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &bitmask_dropout_eps); } diff --git a/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc b/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc index 8a37ef921fd2b..3864baa7c16e2 100644 --- a/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc @@ -33,10 +33,9 @@ static void RunAttentionTest( const std::vector* value_cache = nullptr, const std::initializer_list* key_padding_mask_data = nullptr) { bool enable_cuda = HasCudaEnvironment(0); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_cpu = false; - if (enable_cpu || enable_cuda || enable_rocm) { + if (enable_cpu || enable_cuda) { OpTester tester("DecoderAttention", 1, onnxruntime::kMSDomain); tester.AddAttribute("num_heads", static_cast(num_heads)); tester.AddAttribute("mask_filter_value", static_cast(-10000.0f)); @@ -103,9 +102,7 @@ static void RunAttentionTest( if (enable_cuda) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } } diff --git a/onnxruntime/test/contrib_ops/element_wise_ops_test.cc b/onnxruntime/test/contrib_ops/element_wise_ops_test.cc index 38659fbd9f2b9..3fd27e2ead7c6 100644 --- a/onnxruntime/test/contrib_ops/element_wise_ops_test.cc +++ b/onnxruntime/test/contrib_ops/element_wise_ops_test.cc @@ -109,7 +109,7 @@ TEST(BiasGeluTest, Float) { RunBiasGeluTestFloat({2, 2333}, {2333}); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_DML) || defined(USE_WEBGPU) static void RunBiasGeluTestHalf(const std::vector& input_dims, const std::vector& bias_dims) { RandomValueGenerator random{2333}; std::vector input_data = random.Uniform(input_dims, -1.0f, 1.0f); @@ -147,7 +147,7 @@ TEST(BiasGeluTest, MLFloat16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) static void RunBiasGeluTestBFloat16(const std::vector& input_dims, const std::vector& bias_dims) { RandomValueGenerator random{2333}; std::vector input_data = random.Uniform(input_dims, 0.5f, 1.5f); @@ -164,8 +164,6 @@ static void RunBiasGeluTestBFloat16(const std::vector& input_dims, cons std::vector> execution_providers; #if defined(USE_CUDA) execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif defined(USE_DNNL) execution_providers.push_back(DefaultDnnlExecutionProvider()); #elif defined(USE_DML) @@ -197,7 +195,7 @@ TEST(BiasGeluTest, BFloat16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(MathOpTest, ComplexMul) { std::vector input_a_data = { -0.5f, 0.6f}; @@ -220,8 +218,6 @@ TEST(MathOpTest, ComplexMul) { std::vector> execution_providers; #if defined(USE_CUDA) execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -248,8 +244,6 @@ TEST(MathOpTest, ComplexMulConj) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -276,8 +270,6 @@ TEST(MathOpTest, ComplexMul_fp16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -304,8 +296,6 @@ TEST(MathOpTest, ComplexMulConj_fp16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc index 9ecbb04ebccca..6cd84fc55ea86 100644 --- a/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/embed_layer_norm_op_test.cc @@ -17,11 +17,10 @@ static void RunTest(const embedlayernorm::OpData& data, int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = DefaultRocmExecutionProvider().get() != nullptr; bool enable_dml = DefaultDmlExecutionProvider().get() != nullptr; bool enable_cpu = !use_float16; - if (enable_cpu || enable_cuda || enable_dml || enable_rocm) { + if (enable_cpu || enable_cuda || enable_dml) { // Input and output shapes // Input 0 - input_ids : (batch_size, sequence_size) // Input 1 - segment_ids : (batch_size, sequence_size) @@ -149,10 +148,6 @@ static void RunTest(const embedlayernorm::OpData& data, std::vector> execution_providers; execution_providers.push_back(DefaultCudaExecutionProvider()); tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } else if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider()); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } else if (enable_dml) { std::vector> execution_providers; execution_providers.push_back(DefaultDmlExecutionProvider()); diff --git a/onnxruntime/test/contrib_ops/fastgelu_op_test.cc b/onnxruntime/test/contrib_ops/fastgelu_op_test.cc index 497b8b5fd6cc7..3490516f32099 100644 --- a/onnxruntime/test/contrib_ops/fastgelu_op_test.cc +++ b/onnxruntime/test/contrib_ops/fastgelu_op_test.cc @@ -41,7 +41,7 @@ const std::vector GetExpectedResult(const std::vector& input_data, return ComputeGelu(add_bias_data); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) static void RunFastGeluGpuTest(const std::vector& input_data, const std::vector& bias_data, const std::vector& output_data, const std::vector& input_dims, const std::vector& bias_dims, const std::vector& output_dims, @@ -73,8 +73,6 @@ static void RunFastGeluGpuTest(const std::vector& input_data, const std:: std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_WEBGPU execution_providers.push_back(DefaultWebGpuExecutionProvider()); #endif @@ -144,7 +142,7 @@ static void RunFastGeluTest( std::vector input_dims = {batch_size, sequence_length, hidden_size}; std::vector bias_dims = {hidden_size}; std::vector output_dims = input_dims; -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) RunFastGeluGpuTest(input_data, bias_data, output_data, input_dims, bias_dims, output_dims, has_bias); #endif RunFastGeluCpuTest(input_data, bias_data, output_data, input_dims, bias_dims, output_dims, has_bias); @@ -247,8 +245,8 @@ TEST(FastGeluTest, FastGeluWithoutBiasFloat32) { RunFastGeluTest(input_data, bias_data, batch_size, sequence_length, hidden_size); } -// CUDA, ROCm and WebGPU only for Float16 type. -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +// CUDA and WebGPU only for Float16 type. +#if defined(USE_CUDA) || defined(USE_WEBGPU) TEST(FastGeluTest, FastGeluWithBiasFloat16_2) { int batch_size = 1; int sequence_length = 2; @@ -385,8 +383,8 @@ TEST(FastGeluTest, FastGeluWithoutBiasFloat16_8) { } #endif -// CUDA and ROCm only for BFloat16 type. -#if defined(USE_CUDA) || defined(USE_ROCM) +// CUDA only for BFloat16 type. +#if defined(USE_CUDA) TEST(FastGeluTest, FastGeluWithBias_BFloat16) { #ifdef USE_CUDA int min_cuda_architecture = 800; @@ -433,15 +431,13 @@ TEST(FastGeluTest, FastGeluWithBias_BFloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } #endif -// CUDA and ROCm only for double type. -#if defined(USE_CUDA) || defined(USE_ROCM) +// CUDA only for double type. +#if defined(USE_CUDA) TEST(FastGeluTest, FastGeluWithBias_Double) { OpTester tester("FastGelu", 1, onnxruntime::kMSDomain); @@ -471,8 +467,6 @@ TEST(FastGeluTest, FastGeluWithBias_Double) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/onnxruntime/test/contrib_ops/fft_op_test.cc b/onnxruntime/test/contrib_ops/fft_op_test.cc index 7a6b6cca6425a..1a75f2b12d5eb 100644 --- a/onnxruntime/test/contrib_ops/fft_op_test.cc +++ b/onnxruntime/test/contrib_ops/fft_op_test.cc @@ -8,15 +8,12 @@ namespace onnxruntime { namespace test { TEST(ContribOpTest, Rfft) { - if (DefaultCudaExecutionProvider() == nullptr && DefaultRocmExecutionProvider() == nullptr) return; + if (DefaultCudaExecutionProvider() == nullptr) return; std::vector> execution_providers; if (DefaultCudaExecutionProvider() != nullptr) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (DefaultRocmExecutionProvider() != nullptr) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } OpTester test("Rfft", 1, onnxruntime::kMSDomain); test.AddAttribute("signal_ndim", static_cast(1)); @@ -30,15 +27,12 @@ TEST(ContribOpTest, Rfft) { } TEST(ContribOpTest, Irfft) { - if (DefaultCudaExecutionProvider() == nullptr && DefaultRocmExecutionProvider() == nullptr) return; + if (DefaultCudaExecutionProvider() == nullptr) return; std::vector> execution_providers; if (DefaultCudaExecutionProvider() != nullptr) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (DefaultRocmExecutionProvider() != nullptr) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } OpTester test("Irfft", 1, onnxruntime::kMSDomain); test.AddAttribute("signal_ndim", static_cast(1)); diff --git a/onnxruntime/test/contrib_ops/fused_conv_test.cc b/onnxruntime/test/contrib_ops/fused_conv_test.cc index 0dd69a49972e8..9df222db43501 100644 --- a/onnxruntime/test/contrib_ops/fused_conv_test.cc +++ b/onnxruntime/test/contrib_ops/fused_conv_test.cc @@ -32,17 +32,14 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes, const vector& expected_output_shape, bool disable_cpu = false, bool disable_cuda = false, - bool disable_rocm = false, bool disable_webgpu = false, bool use_float16 = false, bool weight_is_initializer = false) { bool enable_cuda = HasCudaEnvironment(0) && !use_float16 && !disable_cuda; - // Only ROCm EP supports float16. - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()) && !disable_rocm; bool enable_webgpu = (nullptr != DefaultWebGpuExecutionProvider().get()) && !disable_webgpu; bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get()) && !use_float16 && !disable_cpu; - if (enable_cuda || enable_rocm || enable_cpu || enable_webgpu) { + if (enable_cuda || enable_cpu || enable_webgpu) { OpTester test("FusedConv", 1, onnxruntime::kMSDomain); test.AddAttribute("group", attributes.group); test.AddAttribute("kernel_shape", attributes.kernel_shape); @@ -94,10 +91,6 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes, execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } - if (enable_webgpu) { execution_providers.push_back(DefaultWebGpuExecutionProvider()); } @@ -116,16 +109,15 @@ void RunConvOp(const ConvOpAndTestAttributes& attributes, const vector& expected_output_shape, bool disable_cpu = false, bool disable_cuda = false, - bool disable_rocm = false, bool disable_webgpu = false) { bool weight_is_initializer = false; bool use_float16 = false; TestConvOp(attributes, inputs, input_shapes, expected_output, expected_output_shape, - disable_cpu, disable_cuda, disable_rocm, disable_webgpu, use_float16, weight_is_initializer); + disable_cpu, disable_cuda, disable_webgpu, use_float16, weight_is_initializer); use_float16 = true; TestConvOp(attributes, inputs, input_shapes, expected_output, expected_output_shape, - disable_cpu, disable_cuda, disable_rocm, disable_webgpu, use_float16, weight_is_initializer); + disable_cpu, disable_cuda, disable_webgpu, use_float16, weight_is_initializer); } TEST(FusedConvTest, Conv2D_HardSigmoid) { @@ -146,7 +138,7 @@ TEST(FusedConvTest, Conv2D_HardSigmoid) { vector W_shape = {2, 1, 2, 2}; vector Y_shape = {1, 2, 2, 2}; auto expected_vals = {0.8f, 0.9f, 1.0f, 1.0f, 0.2f, 0.1f, 0.0f, 0.0f}; - RunConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, false, true, true, true); + RunConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, false, true, true); } TEST(FusedConvTest, Conv2D_Relu) { @@ -191,7 +183,7 @@ TEST(FusedConvTest, Conv2D_Bias_Relu) { RunConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(FusedConvTest, Conv2D_Bias_Z_Relu) { ConvOpAndTestAttributes attrs = { @@ -214,7 +206,7 @@ TEST(FusedConvTest, Conv2D_Bias_Z_Relu) { vector Z = {-1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}; vector Z_shape = {1, 2, 2, 2}; auto expected_vals = {12.0f, 17.0f, 25.0f, 29.0f, 11.0f, 15.0f, 23.0f, 28.0f}; - RunConvOp(attrs, {X, W, B, Z}, {X_shape, W_shape, B_shape, Z_shape}, expected_vals, Y_shape, true, false, false); + RunConvOp(attrs, {X, W, B, Z}, {X_shape, W_shape, B_shape, Z_shape}, expected_vals, Y_shape, true, false); } #endif @@ -240,7 +232,7 @@ TEST(FusedConvTest, Cpu_Conv2D_Bias_Z_Relu) { vector Z = {-1.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f}; vector Z_shape = {1, 2, 2, 2}; auto expected_vals = {12.0f, 17.0f, 25.0f, 29.0f, 11.0f, 15.0f, 23.0f, 28.0f}; - RunConvOp(attrs, {X, W, B, Z}, {X_shape, W_shape, B_shape, Z_shape}, expected_vals, Y_shape, false, true, true, true); + RunConvOp(attrs, {X, W, B, Z}, {X_shape, W_shape, B_shape, Z_shape}, expected_vals, Y_shape, false, true, true); } #endif diff --git a/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc b/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc index b1762d16795d1..8b15ac5300a82 100644 --- a/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc +++ b/onnxruntime/test/contrib_ops/fused_matmul_op_test.cc @@ -221,7 +221,7 @@ TEST(FusedMatMulOpTest, FloatTypeNoTranspose) { RunFusedMatMulTest("FusedMatMul", 1); } -#if defined(USE_CUDA) || defined(USE_ROCM) // double support only implemented in CUDA/ROCM kernel +#if defined(USE_CUDA) // double support only implemented in CUDA kernel TEST(FusedMatMulOpTest, DoubleTypeNoTranspose) { RunFusedMatMulTest("FusedMatMul", 1); } @@ -270,7 +270,7 @@ TEST(FusedMatMulOpTest, FloatTypeTransposeBatch) { RunFusedMatMulTest("FusedMatMul", 1, true, true, true, true); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_DML) TEST(FusedMatMulOpTest, Float16_NoTranspose) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -315,7 +315,7 @@ TEST(FusedMatMulOpTest, Float16_NoTranspose) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST(FusedMatMulOpTest, BFloat16_NoTranspose) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -371,8 +371,6 @@ TEST(FusedMatMulOpTest, BFloat16_NoTranspose) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif diff --git a/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc b/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc deleted file mode 100644 index 6b67b648fd9b2..0000000000000 --- a/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include -#include "core/platform/threadpool.h" -#include "core/util/math.h" -#include "core/util/thread_utils.h" -#include "test/common/cuda_op_test_utils.h" -#include "test/common/tensor_op_test_utils.h" -#include "test/providers/provider_test_utils.h" - -namespace onnxruntime { -namespace test { -namespace gemmfastgelu { - -#if defined(USE_ROCM) -namespace { - -const onnxruntime::RunOptions run_options = []() { - onnxruntime::RunOptions options{}; - ORT_THROW_IF_ERROR(options.config_options.AddConfigEntry(kOpTesterRunOptionsConfigTestTunableOp, "true")); - return options; -}(); - -const constexpr auto run_with_tunable_op = &run_options; - -} // namespace - -static void RunGemmFastGeluGpuTest(const std::vector& input_data, const std::vector& weight_data, - const std::vector& bias_data, const std::vector& output_data, - const std::vector& input_dims, const std::vector& weight_dims, - const std::vector& bias_dims, const std::vector& output_dims, - bool has_bias, bool use_float16 = false) { - OpTester tester("GemmFastGelu", 1, onnxruntime::kMSDomain); - - if (use_float16) { - tester.AddInput("X", input_dims, ToFloat16(input_data)); - tester.AddInput("W", weight_dims, ToFloat16(weight_data)); - if (has_bias) { - tester.AddInput("bias", bias_dims, ToFloat16(bias_data)); - } - tester.AddOutput("Y", output_dims, ToFloat16(output_data)); - } else { - tester.AddInput("X", input_dims, input_data); - tester.AddInput("W", weight_dims, weight_data); - if (has_bias) { - tester.AddInput("bias", bias_dims, bias_data); - } - tester.AddOutput("Y", output_dims, output_data); - } - - tester.SetOutputTolerance(use_float16 ? 0.005f : 0.0025f); - - tester.Config(run_with_tunable_op) - .RunWithConfig(); -} - -TEST(GemmFastGeluTest, GemmFastGeluWithoutBiasFloat32) { - int batch_size = 1; - int sequence_length = 2; - int hidden_size = 4; - int dense_size = 6; - - std::vector input_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f}; - - std::vector weight_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f, - 0.7f, -0.5f, 0.7f, 1.2f, - 0.3f, 0.1f, 0.8f, -1.6f, - 0.9f, -0.1f, 3.0f, 2.f, - 0.4f, -0.7f, -0.3f, 0.6f}; - - std::vector bias_data = {}; - - std::vector output_data = { - 3.4894f, 1.8455f, 0.0260f, 0.2229f, -0.1003f, 0.0902f, - -0.1323f, -0.0953f, 0.0778f, 0.2152f, 0.6715f, -0.0240f}; - - std::vector input_dims = {batch_size, sequence_length, hidden_size}; - std::vector weight_dims = {hidden_size, dense_size}; - std::vector bias_dims = {dense_size}; - std::vector output_dims = {batch_size, sequence_length, dense_size}; - - RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, - input_dims, weight_dims, bias_dims, output_dims, - false); -} - -TEST(GemmFastGeluTest, GemmFastGeluWithBiasFloat32) { - int batch_size = 1; - int sequence_length = 2; - int hidden_size = 4; - int dense_size = 6; - - std::vector input_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f}; - - std::vector weight_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f, - 0.7f, -0.5f, 0.7f, 1.2f, - 0.3f, 0.1f, 0.8f, -1.6f, - 0.9f, -0.1f, 3.0f, 2.f, - 0.4f, -0.7f, -0.3f, 0.6f}; - - std::vector bias_data = { - -0.5f, 0.6f, 1.2f, 2.1f, -0.6f, 0.4f}; - - std::vector output_data = { - 2.9862f, 2.4849f, 1.1177f, 2.4329f, -0.1681f, 0.3988f, - -0.0702f, -0.1633f, 1.2190f, 2.4225f, 0.1428f, 0.2229f}; - - std::vector input_dims = {batch_size, sequence_length, hidden_size}; - std::vector weight_dims = {hidden_size, dense_size}; - std::vector bias_dims = {dense_size}; - std::vector output_dims = {batch_size, sequence_length, dense_size}; - - RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, - input_dims, weight_dims, bias_dims, output_dims, - true); -} - -TEST(GemmFastGeluTest, GemmFastGeluWithoutBiasFloat16) { - int batch_size = 1; - int sequence_length = 2; - int hidden_size = 4; - int dense_size = 6; - - std::vector input_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f}; - - std::vector weight_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f, - 0.7f, -0.5f, 0.7f, 1.2f, - 0.3f, 0.1f, 0.8f, -1.6f, - 0.9f, -0.1f, 3.0f, 2.f, - 0.4f, -0.7f, -0.3f, 0.6f}; - - std::vector bias_data = {}; - - std::vector output_data = { - 3.4902f, 1.8467f, 0.0259f, 0.2227f, -0.1005f, 0.0901f, - -0.1324f, -0.0955f, 0.0778f, 0.2156f, 0.6714f, -0.0241f}; - - std::vector input_dims = {batch_size, sequence_length, hidden_size}; - std::vector weight_dims = {hidden_size, dense_size}; - std::vector bias_dims = {dense_size}; - std::vector output_dims = {batch_size, sequence_length, dense_size}; - - RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, - input_dims, weight_dims, bias_dims, output_dims, - false, true); -} - -TEST(GemmFastGeluTest, GemmFastGeluWithBiasFloat16) { - int batch_size = 1; - int sequence_length = 2; - int hidden_size = 4; - int dense_size = 6; - - std::vector input_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f}; - - std::vector weight_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f, - 0.7f, -0.5f, 0.7f, 1.2f, - 0.3f, 0.1f, 0.8f, -1.6f, - 0.9f, -0.1f, 3.0f, 2.f, - 0.4f, -0.7f, -0.3f, 0.6f}; - - std::vector bias_data = { - -0.5f, 0.6f, 1.2f, 2.1f, -0.6f, 0.4f}; - - std::vector output_data = { - 2.9883f, 2.4844f, 1.1182f, 2.4316f, -0.1680f, 0.3984f, - -0.0701f, -0.1633f, 1.2178f, 2.4219f, 0.1426f, 0.2227f}; - - std::vector input_dims = {batch_size, sequence_length, hidden_size}; - std::vector weight_dims = {hidden_size, dense_size}; - std::vector bias_dims = {dense_size}; - std::vector output_dims = {batch_size, sequence_length, dense_size}; - - RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, - input_dims, weight_dims, bias_dims, output_dims, - true, true); -} - -TEST(GemmFastGeluTest, GemmFastGeluWithBias_bfloat16) { - OpTester tester("GemmFastGelu", 1, onnxruntime::kMSDomain); - - int batch_size = 1; - int sequence_length = 2; - int hidden_size = 4; - int dense_size = 6; - - std::vector input_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f}; - - std::vector weight_data = { - 0.8f, -0.5f, 0.0f, 1.f, - 0.5f, 0.2f, 0.3f, -0.6f, - 0.7f, -0.5f, 0.7f, 1.2f, - 0.3f, 0.1f, 0.8f, -1.6f, - 0.9f, -0.1f, 3.0f, 2.f, - 0.4f, -0.7f, -0.3f, 0.6f}; - - std::vector bias_data = { - -0.5f, 0.6f, 1.2f, 2.1f, -0.6f, 0.4f}; - - std::vector output_data = { - 2.9883f, 2.4844f, 1.1182f, 2.4316f, -0.1680f, 0.3984f, - -0.0701f, -0.1633f, 1.2178f, 2.4219f, 0.1426f, 0.2227f}; - - std::vector input_dims = {batch_size, sequence_length, hidden_size}; - std::vector weight_dims = {hidden_size, dense_size}; - std::vector bias_dims = {dense_size}; - std::vector output_dims = {batch_size, sequence_length, dense_size}; - - std::vector f_X = FloatsToBFloat16s(input_data); - std::vector f_W = FloatsToBFloat16s(weight_data); - std::vector f_B = FloatsToBFloat16s(bias_data); - std::vector f_Y = FloatsToBFloat16s(output_data); - - tester.AddInput("X", input_dims, f_X); - tester.AddInput("W", weight_dims, f_W); - tester.AddInput("bias", bias_dims, f_B); - tester.AddOutput("Y", output_dims, f_Y); - - tester.Config(run_with_tunable_op) - .RunWithConfig(); -} -#endif - -} // namespace gemmfastgelu -} // namespace test -} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/greedy_search_test.cc b/onnxruntime/test/contrib_ops/greedy_search_test.cc index be72fbd460c9b..04a57dd9c9b2c 100644 --- a/onnxruntime/test/contrib_ops/greedy_search_test.cc +++ b/onnxruntime/test/contrib_ops/greedy_search_test.cc @@ -60,13 +60,8 @@ TEST(GreedySearchTest, GptGreedySearchFp16_VocabPadded) { #else bool is_cuda = false; #endif -#ifdef USE_ROCM - bool is_rocm = true; -#else - bool is_rocm = false; -#endif - if (is_cuda || is_rocm) { + if (is_cuda) { Ort::SessionOptions session_options; #ifdef USE_CUDA if (is_cuda) { @@ -142,13 +137,8 @@ TEST(GreedySearchTest, GptGreedySearchFp32) { #else bool is_cuda = false; #endif -#ifdef USE_ROCM - bool is_rocm = true; -#else - bool is_rocm = false; -#endif - if (is_cuda || is_rocm) { + if (is_cuda) { Ort::SessionOptions session_options; #ifdef USE_CUDA if (is_cuda) { diff --git a/onnxruntime/test/contrib_ops/group_norm_op_test.cc b/onnxruntime/test/contrib_ops/group_norm_op_test.cc index fdc546441676b..5227509368f45 100644 --- a/onnxruntime/test/contrib_ops/group_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/group_norm_op_test.cc @@ -730,20 +730,17 @@ TEST(GroupNormTest, GroupNorm_128) { // Test float16, without activation int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()); std::array channels_last_values = {-1, 0, 1}; for (const int channels_last : channels_last_values) { - if (enable_cuda || enable_rocm || enable_dml) { + if (enable_cuda || enable_dml) { std::vector> execution_providers; if (enable_cuda && channels_last != 0) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm && channels_last != 0) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); } @@ -784,14 +781,12 @@ TEST(GroupNormTest, GroupNorm_128) { // Test float32, with activation enable_cuda = HasCudaEnvironment(0); - if (enable_cuda || enable_rocm || enable_dml) { + if (enable_cuda || enable_dml) { std::vector> execution_providers; if (enable_cuda && channels_last != 0) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm && channels_last != 0) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); } diff --git a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc index 0d4fc5af68b4f..d08df321a963b 100644 --- a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc @@ -21,7 +21,7 @@ using namespace std; namespace onnxruntime { namespace test { -// Some feature (like broadcast support) are implemented in CPU and CUDA/ROCM provider only. A helper to run tests. +// Some feature (like broadcast support) are implemented in CPU and CUDA provider only. A helper to run tests. void RunTestOnCpuAndCuda(OpTester& test, const std::string& expected_failure_msg = "") { auto expected_result = expected_failure_msg.empty() ? OpTester::ExpectResult::kExpectSuccess @@ -33,13 +33,11 @@ void RunTestOnCpuAndCuda(OpTester& test, const std::string& expected_failure_msg constexpr int min_cuda_architecture = 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); - if (enable_cuda || enable_rocm) { + + if (enable_cuda) { std::vector> gpu_execution_provider; if (enable_cuda) { gpu_execution_provider.push_back(DefaultCudaExecutionProvider()); - } else if (enable_rocm) { - gpu_execution_provider.push_back(DefaultRocmExecutionProvider()); } if (gpu_execution_provider.size() > 0) { diff --git a/onnxruntime/test/contrib_ops/layer_norm_test.cc b/onnxruntime/test/contrib_ops/layer_norm_test.cc index 46082e1b0cd31..75e1e0856bc7e 100644 --- a/onnxruntime/test/contrib_ops/layer_norm_test.cc +++ b/onnxruntime/test/contrib_ops/layer_norm_test.cc @@ -6,7 +6,7 @@ namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_DML) || defined(USE_WEBGPU) constexpr auto k_epsilon_default = 1e-5f; constexpr auto k_random_data_min = -10.0f; constexpr auto k_random_data_max = 10.0f; @@ -80,8 +80,6 @@ static void TestLayerNorm(const std::vector& x_dims, #ifdef USE_CUDA test.CompareWithCPU(kCudaExecutionProvider); -#elif USE_ROCM - test.CompareWithCPU(kRocmExecutionProvider); #elif USE_DML test.CompareWithCPU(kDmlExecutionProvider); #elif USE_WEBGPU diff --git a/onnxruntime/test/contrib_ops/longformer_attention_op_test.cc b/onnxruntime/test/contrib_ops/longformer_attention_op_test.cc index 3e5c9a10f32b5..c7c03ed7080ae 100644 --- a/onnxruntime/test/contrib_ops/longformer_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/longformer_attention_op_test.cc @@ -29,7 +29,6 @@ static void RunAttentionTest( int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_cpu = false; if (enable_cpu || enable_cuda) { OpTester tester("LongformerAttention", 1, onnxruntime::kMSDomain); @@ -69,9 +68,7 @@ static void RunAttentionTest( if (enable_cuda) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } + if (enable_cpu) { execution_providers.push_back(DefaultCpuExecutionProvider()); } diff --git a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc index e5cfb946999a6..21e2003bf9acf 100644 --- a/onnxruntime/test/contrib_ops/matmul_4bits_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_4bits_test.cc @@ -502,7 +502,7 @@ TEST(MatMulNBits, LegacyShape_4b) { #endif #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_DML) || defined(USE_WEBGPU) namespace { // Legacy test function. @@ -538,10 +538,6 @@ void RunTest(int64_t M, int64_t N, int64_t K, int64_t block_size, bool has_zerop execution_providers.push_back(DefaultCudaExecutionProvider()); RunTest(opts, std::move(execution_providers)); #endif -#ifdef USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); - RunTest(opts, std::move(execution_providers)); -#endif #ifdef USE_DML execution_providers.push_back(DefaultDmlExecutionProvider()); RunTest(opts, std::move(execution_providers)); @@ -551,9 +547,6 @@ void RunTest(int64_t M, int64_t N, int64_t K, int64_t block_size, bool has_zerop RunTest(opts, std::move(execution_providers)); #endif } else { -#ifdef USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); -#endif #ifdef USE_WEBGPU ConfigOptions config_options{}; ORT_ENFORCE(config_options.AddConfigEntry(webgpu::options::kMaxStorageBufferBindingSize, "134217728").IsOK()); @@ -737,7 +730,7 @@ TEST(MatMulNBits, BFloat16_Int4_NoZeroPoint) { } #endif -#endif // defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#endif // defined(USE_CUDA) || defined(USE_DML) } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/multihead_attention_op_test.cc b/onnxruntime/test/contrib_ops/multihead_attention_op_test.cc index a7d5f15698f0c..c740959105977 100644 --- a/onnxruntime/test/contrib_ops/multihead_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/multihead_attention_op_test.cc @@ -9,18 +9,6 @@ #include "test/util/include/scoped_env_vars.h" #include "test/contrib_ops/attention_op_test_helper.h" -#if defined(USE_ROCM) && defined(USE_COMPOSABLE_KERNEL) && !defined(USE_MIGRAPHX) -#define DISABLE_ROCM false -#else -#define DISABLE_ROCM true -#endif - -#if defined(USE_ROCM) -#define ROCM_GTEST_SKIP(message) GTEST_SKIP_(message) -#else -#define ROCM_GTEST_SKIP(message) -#endif - namespace onnxruntime { namespace test { @@ -57,30 +45,17 @@ static void RunMultiHeadAttentionTest( bool disable_cpu = false, // some cases not supported in cpu right now. bool disable_cuda = false, bool disable_webgpu = false, - bool disable_rocm = DISABLE_ROCM, // not supported in rocm right now. bool disable_dml = false) { kv_sequence_length = (kv_sequence_length == 0 ? sequence_length : kv_sequence_length); int past_sequence_length = (past_seq_len_data.size() == 0) ? 0 : past_seq_len_data[0]; int min_cuda_architecture = use_float16 ? 750 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture) && !disable_cuda; - // rocm mha is required to work with TunableOp Enabled - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider(/*test_tunable_op=*/true).get()) && !disable_rocm; bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get()) && !use_float16 && !disable_cpu; bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()) && !disable_dml; bool enable_webgpu = (nullptr != DefaultWebGpuExecutionProvider().get()) && !disable_webgpu; - if (enable_rocm && !use_float16) { - LOGS_DEFAULT(WARNING) << "ROCm MHA only have kernel for half datatype implemented, skip float datatype tests"; - enable_rocm = false; - } - - if (enable_rocm && !bias_data.empty()) { - LOGS_DEFAULT(WARNING) << "ROCm MHA does not support qkv_bias, skip qkv_bias tests"; - enable_rocm = false; - } - - if (enable_cpu || enable_cuda || enable_rocm || enable_dml || enable_webgpu) { + if (enable_cpu || enable_cuda || enable_dml || enable_webgpu) { OpTester tester("MultiHeadAttention", 1, onnxruntime::kMSDomain); tester.AddAttribute("num_heads", static_cast(num_heads)); tester.AddAttribute("unidirectional", static_cast(is_unidirectional ? 1 : 0)); @@ -301,12 +276,6 @@ static void RunMultiHeadAttentionTest( tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } - if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/true)); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } - if (enable_cpu) { std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -361,7 +330,6 @@ static void RunMultiHeadAttentionKernel( bool disable_cpu = false, // some cases not supported in cpu right now. bool disable_cuda = false, bool disable_webgpu = false, - bool disable_rocm = DISABLE_ROCM, bool disable_dml = false) { if (kernel_type == AttentionKernelType::AttentionKernel_Default) { ScopedEnvironmentVariables scoped_env_vars{ @@ -377,7 +345,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); return; } @@ -395,7 +363,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); return; } @@ -413,7 +381,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); return; } @@ -432,7 +400,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); return; } #endif @@ -452,7 +420,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } if (kernel_type == AttentionKernelType::AttentionKernel_CudnnFlashAttention) { @@ -470,7 +438,7 @@ static void RunMultiHeadAttentionKernel( present_key_data, present_value_data, key_padding_mask_data, mask_type, output_data, output_qk_data, num_heads, batch_size, sequence_length, kv_sequence_length, hidden_size, v_hidden_size, num_beams, max_sequence_length, is_static_kv, buffer_share, use_float16, - is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + is_unidirectional, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } } @@ -479,7 +447,6 @@ enum RunMultiHeadAttentionTestToggles : uint32_t { DISABLE_CPU = 1 << 0, DISABLE_CUDA = 1 << 1, DISABLE_WEBGPU = 1 << 2, - DISABLE_ROCM_MHA = 1 << 3, DISABLE_DML = 1 << 4, }; inline RunMultiHeadAttentionTestToggles operator|(RunMultiHeadAttentionTestToggles a, RunMultiHeadAttentionTestToggles b) { @@ -494,7 +461,6 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, bool disable_cpu = toggles & DISABLE_CPU; bool disable_cuda = toggles & DISABLE_CUDA; bool disable_webgpu = toggles & DISABLE_WEBGPU; - bool disable_rocm = toggles & DISABLE_ROCM_MHA; bool disable_dml = toggles & DISABLE_DML; if (data.fp32_output_data.size() > 0) { @@ -508,7 +474,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp32_output_data, data.fp32_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } #if USE_MEMORY_EFFICIENT_ATTENTION @@ -522,7 +488,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp32_output_data, data.fp32_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } } #endif @@ -534,7 +500,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp32_output_data, data.fp32_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } if (data.fp16_output_data.size() > 0) { @@ -547,7 +513,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp16_output_data, data.fp16_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } kernel_type = AttentionKernelType::AttentionKernel_TrtFusedAttention; @@ -558,7 +524,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp16_output_data, data.fp16_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } #if USE_MEMORY_EFFICIENT_ATTENTION @@ -570,7 +536,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp16_output_data, data.fp16_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } #endif @@ -582,7 +548,7 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp16_output_data, data.fp16_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } kernel_type = AttentionKernelType::AttentionKernel_Default; @@ -592,14 +558,13 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, data.present_key_data, data.present_value_data, data.key_padding_mask_data, data.mask_type, data.fp16_output_data, data.fp16_output_qk_data, kernel_type, data.num_heads, data.batch_size, data.sequence_length, data.kv_sequence_length, data.hidden_size, data.v_hidden_size, data.num_beams, data.max_sequence_length, data.is_static_kv, data.buffer_share, use_float16, - false, disable_cpu, disable_cuda, disable_webgpu, disable_rocm, disable_dml); + false, disable_cpu, disable_cuda, disable_webgpu, disable_dml); } } // Test fused cross attention kernel // It requires head_size > 32 and head_size <= 64 for T4 GPU; hidden_size == v_hidden_size. TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize40) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_HeadSize40(data); RunMultiHeadAttentionTests(data); @@ -609,7 +574,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize40) { } TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize32_RightSidePadding_Mask1D) { - ROCM_GTEST_SKIP("ROCm MHA does not support mask type of MASK_1D_KEY_SEQ_LEN"); AttentionTestData data; GetCrossAttentionData_Batch2_HeadSize32_RightSidePadding(data, true); RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_WEBGPU); @@ -619,7 +583,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize32_RightSidePadding_M } TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize32_RightSidePadding_Mask2D) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_Batch2_HeadSize32_RightSidePadding(data, false); RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_WEBGPU); @@ -629,7 +592,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize32_RightSidePadding_M } TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize32_LeftSidePadding_Mask2D) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_Batch1_HeadSize32_LeftSidePadding(data); RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_WEBGPU); @@ -639,14 +601,12 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize32_LeftSidePadding_Ma } TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize32_NoBias_NoMask_PackedKV) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedKV(data); RunMultiHeadAttentionTests(data, DISABLE_WEBGPU); } TEST(MultiHeadAttentionTest, SelfAttention_Batch2_HeadSize32_NoBias_NoMask_PackedQKV) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetSelfAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedQKV(data); RunMultiHeadAttentionTests(data, DISABLE_WEBGPU); @@ -654,7 +614,6 @@ TEST(MultiHeadAttentionTest, SelfAttention_Batch2_HeadSize32_NoBias_NoMask_Packe // This tests qk_head_size != v_head_size TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize16_8) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_HeadSize16_8(data); RunMultiHeadAttentionTests(data); @@ -664,7 +623,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize16_8) { } TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize16) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_HeadSize16(data); RunMultiHeadAttentionTests(data); @@ -674,7 +632,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize16) { } TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize8) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); AttentionTestData data; GetCrossAttentionData_HeadSize8_NoBias(data); RunMultiHeadAttentionTests(data, DISABLE_CUDA); @@ -684,7 +641,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_Batch1_HeadSize8) { // Bug #50220930 #ifndef USE_DML TEST(MultiHeadAttentionTest, CrossAttentionWithPast) { - ROCM_GTEST_SKIP("ROCm MHA only support head_size >= 8"); AttentionTestData data; GetCrossAttentionDataWithPast(data); RunMultiHeadAttentionTests(data, DISABLE_WEBGPU); @@ -692,22 +648,18 @@ TEST(MultiHeadAttentionTest, CrossAttentionWithPast) { #endif TEST(MultiHeadAttentionTest, SelfAttention_WithPast_WithAttnBias_ForT5) { - ROCM_GTEST_SKIP("ROCm MHA only support head_size >= 8"); AttentionTestData data; GetSelfAttentionData_WithPast_WithAttnBias_ForT5(data); RunMultiHeadAttentionTests(data, DISABLE_CPU); } TEST(MultiHeadAttentionTest, AttentionCutlassRelPosBias) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); - // ROCM_GTEST_SKIP("ROCm does not support cutlass"); AttentionTestData data; GetAttentionDataCutlassAttnBias(data); RunMultiHeadAttentionTests(data, DISABLE_WEBGPU); } TEST(MultiHeadAttentionTest, CrossAttention_DiffSequenceLengths) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); // Whisper decoder cross attention without mask and different sequence lengths for Q and K/V AttentionTestData data; GetCrossAttentionData_DiffSequenceLengths(data); @@ -721,7 +673,6 @@ TEST(MultiHeadAttentionTest, CrossAttention_DiffSequenceLengths) { } TEST(MultiHeadAttentionTest, SelfAttention_WithPastAndPresent_NoMask_NoRelPosBias) { - ROCM_GTEST_SKIP("ROCm MHA skip - missing support for ROCm on Radeon"); // Whisper decoder self attention with past_kv and present_kv AttentionTestData data; GetSelfAttentionData_WithPastAndPresent_NoMask_NoAttnBias(data); @@ -734,7 +685,7 @@ TEST(MultiHeadAttentionTest, SelfAttention_WithPastAndPresent_NoMask_NoRelPosBia RunMultiHeadAttentionTests(data, DISABLE_CUDA); } -// This test is disabled since it is not used in Whisper anymore, and it fails in ROCm. +// This test is disabled since it is not used in Whisper anymore. TEST(MultiHeadAttentionTest, DISABLED_CrossAttention_WithPastPassedInDirectly_NoMask) { // Whisper decoder cross attention with past_kv in place of current KV and no present_kv AttentionTestData data; @@ -749,7 +700,7 @@ TEST(MultiHeadAttentionTest, SelfAttention_PastPresentBufferShare_UsingDMMHAInsi // See onnxruntime/core/graph/contrib_ops/bert_defs.cc for more details AttentionTestData data; GetSelfAttention_PastPresentBufferShare_UsingDMMHAInsideMHA(data); - RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_ROCM_MHA | DISABLE_WEBGPU | DISABLE_DML); + RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_WEBGPU | DISABLE_DML); } TEST(MultiHeadAttentionTest, CrossAttention_DiffSequenceLengths_UsingDMMHAInsideMHA) { @@ -757,7 +708,7 @@ TEST(MultiHeadAttentionTest, CrossAttention_DiffSequenceLengths_UsingDMMHAInside // Used in decoder-with-past's cross-attention layers AttentionTestData data; GetCrossAttention_DiffSequenceLengths_UsingDMMHAInsideMHA(data); - RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_ROCM_MHA | DISABLE_WEBGPU | DISABLE_DML); + RunMultiHeadAttentionTests(data, DISABLE_CPU | DISABLE_WEBGPU | DISABLE_DML); } } // namespace test diff --git a/onnxruntime/test/contrib_ops/ngram_repeat_block_op_test.cc b/onnxruntime/test/contrib_ops/ngram_repeat_block_op_test.cc index 09b98aa50bd7a..f57882473e30b 100644 --- a/onnxruntime/test/contrib_ops/ngram_repeat_block_op_test.cc +++ b/onnxruntime/test/contrib_ops/ngram_repeat_block_op_test.cc @@ -31,13 +31,6 @@ TEST(NGramRepeatBlockTest, NGramSize_3) { execution_providers.push_back(DefaultCudaExecutionProvider()); tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } -#ifdef USE_ROCM - if (nullptr != DefaultRocmExecutionProvider().get()) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider()); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } -#endif std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); diff --git a/onnxruntime/test/contrib_ops/nhwc_conv_op_test.cc b/onnxruntime/test/contrib_ops/nhwc_conv_op_test.cc index e780d35df08bd..850bea4351914 100644 --- a/onnxruntime/test/contrib_ops/nhwc_conv_op_test.cc +++ b/onnxruntime/test/contrib_ops/nhwc_conv_op_test.cc @@ -32,10 +32,9 @@ void TestNhwcConvOp(const NhwcConvOpAndTestAttributes& attributes, int min_cuda_architecture = use_float16 ? 530 : 0; // NHWC implementation doesn't handle W in NHWC layout if it's not an initializer bool enable_cuda = HasCudaEnvironment(min_cuda_architecture) && weight_is_initializer; - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_dml = (nullptr != DefaultDmlExecutionProvider().get()); - if (enable_cuda || enable_rocm || enable_dml) { + if (enable_cuda || enable_dml) { OpTester test("NhwcConv", 1, onnxruntime::kMSDomain); test.AddAttribute("group", attributes.group); test.AddAttribute("kernel_shape", attributes.kernel_shape); @@ -80,10 +79,6 @@ void TestNhwcConvOp(const NhwcConvOpAndTestAttributes& attributes, execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } - if (enable_dml) { execution_providers.push_back(DefaultDmlExecutionProvider()); } diff --git a/onnxruntime/test/contrib_ops/remove_padding_op_test.cc b/onnxruntime/test/contrib_ops/remove_padding_op_test.cc index d1a189de9ad4a..fe415e09fde62 100644 --- a/onnxruntime/test/contrib_ops/remove_padding_op_test.cc +++ b/onnxruntime/test/contrib_ops/remove_padding_op_test.cc @@ -22,14 +22,12 @@ static void RunRemovePadding( int total_tokens, bool use_float16 = false, const bool disable_cpu = true, - const bool disable_cuda = false, - const bool disable_rocm = true) { + const bool disable_cuda = false) { int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture) && !disable_cuda; - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()) && !disable_rocm; bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get()) && !use_float16 && !disable_cpu; - if (enable_cpu || enable_cuda || enable_rocm) { + if (enable_cpu || enable_cuda) { OpTester tester("RemovePadding", 1, onnxruntime::kMSDomain); // shape of inputs: @@ -68,12 +66,6 @@ static void RunRemovePadding( tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } - if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider()); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } - if (enable_cpu) { std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -96,15 +88,14 @@ static void RunRemovePaddingTests( bool use_float16 = false; constexpr bool disable_cpu = true; constexpr bool disable_cuda = false; - constexpr bool disable_rocm = true; RunRemovePadding(input_data, sequence_token_count_data, output_data, token_offset_data, cumulated_seq_len_data, max_token_count, batch_size, sequence_length, hidden_size, total_tokens, - use_float16, disable_cpu, disable_cuda, disable_rocm); + use_float16, disable_cpu, disable_cuda); use_float16 = true; RunRemovePadding(input_data, sequence_token_count_data, output_data, token_offset_data, cumulated_seq_len_data, max_token_count, batch_size, sequence_length, hidden_size, total_tokens, - use_float16, disable_cpu, disable_cuda, disable_rocm); + use_float16, disable_cpu, disable_cuda); } TEST(RemovePaddingTest, RemovePaddingBatch1_NoPadding) { diff --git a/onnxruntime/test/contrib_ops/restore_padding_op_test.cc b/onnxruntime/test/contrib_ops/restore_padding_op_test.cc index c8d49ce465bd6..3fef9857e4032 100644 --- a/onnxruntime/test/contrib_ops/restore_padding_op_test.cc +++ b/onnxruntime/test/contrib_ops/restore_padding_op_test.cc @@ -19,14 +19,12 @@ static void RunRestorePadding( int total_tokens, bool use_float16 = false, const bool disable_cpu = true, - const bool disable_cuda = false, - const bool disable_rocm = true) { + const bool disable_cuda = false) { int min_cuda_architecture = use_float16 ? 530 : 0; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture) && !disable_cuda; - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()) && !disable_rocm; bool enable_cpu = (nullptr != DefaultCpuExecutionProvider().get()) && !use_float16 && !disable_cpu; - if (enable_cpu || enable_cuda || enable_rocm) { + if (enable_cpu || enable_cuda) { OpTester tester("RestorePadding", 1, onnxruntime::kMSDomain); // shape of inputs: @@ -54,12 +52,6 @@ static void RunRestorePadding( tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } - if (enable_rocm) { - std::vector> execution_providers; - execution_providers.push_back(DefaultRocmExecutionProvider()); - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); - } - if (enable_cpu) { std::vector> execution_providers; execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -79,13 +71,12 @@ static void RunRestorePaddingTests( bool use_float16 = false; constexpr bool disable_cpu = true; constexpr bool disable_cuda = false; - constexpr bool disable_rocm = true; RunRestorePadding(input_data, output_data, token_offset_data, batch_size, sequence_length, hidden_size, total_tokens, - use_float16, disable_cpu, disable_cuda, disable_rocm); + use_float16, disable_cpu, disable_cuda); use_float16 = true; RunRestorePadding(input_data, output_data, token_offset_data, batch_size, sequence_length, hidden_size, total_tokens, - use_float16, disable_cpu, disable_cuda, disable_rocm); + use_float16, disable_cpu, disable_cuda); } TEST(RestorePaddingTest, RestorePaddingBatch1_NoPadding) { diff --git a/onnxruntime/test/contrib_ops/sampling_test.cc b/onnxruntime/test/contrib_ops/sampling_test.cc index 69789b84832e0..b9bb1004332db 100644 --- a/onnxruntime/test/contrib_ops/sampling_test.cc +++ b/onnxruntime/test/contrib_ops/sampling_test.cc @@ -18,7 +18,7 @@ namespace onnxruntime { namespace test { #if defined(__linux__) && !defined(__ANDROID__) -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(SamplingTest, Gpt2Sampling_GPU) { std::vector input_ids{ 0, 0, 0, 0, 0, 52, 195, 731, 321, 301, 734, 620, @@ -73,10 +73,6 @@ TEST(SamplingTest, Gpt2Sampling_GPU) { OrtCUDAProviderOptionsV2 cuda_options; cuda_options.use_tf32 = false; session_options.AppendExecutionProvider_CUDA_V2(cuda_options); -#else // USE_ROCM - OrtROCMProviderOptions rocm_options; - // TODO - verify the default settings - session_options.AppendExecutionProvider_ROCM(rocm_options); #endif Ort::Session session(*ort_env, ORT_TSTR("testdata/transformers/tiny_gpt2_sampling.onnx"), session_options); diff --git a/onnxruntime/test/contrib_ops/skip_group_norm_op_test.cc b/onnxruntime/test/contrib_ops/skip_group_norm_op_test.cc index 3e8870892b7c9..c140f18cb9fe3 100644 --- a/onnxruntime/test/contrib_ops/skip_group_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/skip_group_norm_op_test.cc @@ -114,21 +114,16 @@ TEST(SkipGroupNormTest, SkipGroupNorm_with_bias) { int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); std::array channels_last_values = {-1, 1}; for (const int channels_last : channels_last_values) { - if (enable_cuda || enable_rocm) { + if (enable_cuda) { std::vector> execution_providers; if (enable_cuda && channels_last != 0) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm && channels_last != 0) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } - // Don't run the test if no providers are supported if (execution_providers.empty()) { continue; @@ -235,7 +230,6 @@ TEST(SkipGroupNormTest, SkipGroupNorm_no_bias_broadcast_skip) { int min_cuda_architecture = 530; bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); - bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); std::array has_add_out_values = {true, false}; std::array skip_dims = {2, 4}; @@ -243,16 +237,12 @@ TEST(SkipGroupNormTest, SkipGroupNorm_no_bias_broadcast_skip) { constexpr int channels_last = 1; for (const int skip_dim : skip_dims) { for (const bool has_add_out : has_add_out_values) { - if (enable_cuda || enable_rocm) { + if (enable_cuda) { std::vector> execution_providers; if (enable_cuda && channels_last != 0) { execution_providers.push_back(DefaultCudaExecutionProvider()); } - if (enable_rocm && channels_last != 0) { - execution_providers.push_back(DefaultRocmExecutionProvider()); - } - // Don't run the test if no providers are supported if (execution_providers.empty()) { continue; diff --git a/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc b/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc index a1856b70f711f..85538efbefd28 100644 --- a/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc +++ b/onnxruntime/test/contrib_ops/skiplayernorm_op_test.cc @@ -60,7 +60,6 @@ static void RunOneTest( std::string op_type = simplified ? "SkipSimplifiedLayerNormalization" : "SkipLayerNormalization"; - auto rocm_ep = DefaultRocmExecutionProvider(); auto dml_ep = DefaultDmlExecutionProvider(); auto cpu_ep = DefaultCpuExecutionProvider(); auto webgpu_ep = DefaultWebGpuExecutionProvider(); @@ -147,7 +146,6 @@ static void RunOneTest( test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } else if (HasCudaEnvironment(530 /*min_cuda_architecture*/) || dml_ep != nullptr || - rocm_ep != nullptr || webgpu_ep != nullptr) { OpTester test(op_type.c_str(), 1, onnxruntime::kMSDomain); test.AddInput("input", input_dims, ToFloat16(input_data)); @@ -186,8 +184,6 @@ static void RunOneTest( execution_providers.push_back(DefaultWebGpuExecutionProvider()); } else if (dml_ep != nullptr) { execution_providers.push_back(DefaultDmlExecutionProvider()); - } else if (rocm_ep != nullptr) { - execution_providers.push_back(DefaultRocmExecutionProvider()); } else { if (strict) { Ort::CUDAProviderOptions cuda_options; @@ -877,7 +873,6 @@ TEST(SkipLayerNormTest, SkipSimplifiedLayerNormBatch1_Float16) { simplified); } -#if !defined(USE_ROCM) TEST(SkipLayerNormTest, SkipLayerNormBatch2_Skip_Broadcast_No_Batch_Size) { int batch_size = 2; int sequence_length = 2; @@ -987,7 +982,6 @@ TEST(SkipLayerNormTest, SkipLayerNormBatch2_Skip_Broadcast_Batch_Size_1) { broadcast_skip, no_batch_size); } -#endif } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index aca345fccdc01..8b66009c0c72f 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -40,10 +40,6 @@ #ifdef USE_TENSORRT #include "core/providers/tensorrt/tensorrt_provider_options.h" #endif -#ifdef USE_ROCM -#include "core/providers/rocm/rocm_provider_factory.h" -#include "core/providers/rocm/gpu_data_transfer.h" -#endif #include "core/session/allocator_adapters.h" #include "core/session/environment.h" #include "core/session/IOBinding.h" @@ -77,9 +73,6 @@ namespace onnxruntime { #ifdef USE_CUDA ProviderInfo_CUDA& GetProviderInfo_CUDA(); #endif -#ifdef USE_ROCM -ProviderInfo_ROCM& GetProviderInfo_ROCM(); -#endif class FuseAdd : public OpKernel { public: @@ -217,7 +210,7 @@ static void CreateMatMulModel(std::unique_ptr& p_model, Prov if (provider_type == kCpuExecutionProvider) { node.SetExecutionProviderType(provider_type); } else { -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) node.SetExecutionProviderType(provider_type); #endif } @@ -307,7 +300,7 @@ void RunModelWithBindingMatMul(InferenceSession& session_object, // And it can't be used for copying buffer to buffer since the target buffer is still in mapped state. OrtMemoryInfo mem_info(WEBGPU_BUFFER, OrtAllocatorType::OrtDeviceAllocator, OrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT, OrtDevice::VendorIds::NONE, 0)); gpu_alloc = session_object.GetAllocator(mem_info); - } else if (allocation_provider == kCudaExecutionProvider || allocation_provider == kRocmExecutionProvider) { + } else if (allocation_provider == kCudaExecutionProvider) { gpu_alloc = gpu_provider->CreatePreferredAllocators()[0]; } if (enable_graph_capture) { @@ -367,7 +360,7 @@ void RunModelWithBindingMatMul(InferenceSession& session_object, if (is_preallocate_output_vec) { if (allocation_provider == kCpuExecutionProvider) { AllocateMLValue(cpu_alloc, expected_output_dims, &output_ml_value); - } else if (allocation_provider == kCudaExecutionProvider || allocation_provider == kRocmExecutionProvider || allocation_provider == kWebGpuExecutionProvider) { + } else if (allocation_provider == kCudaExecutionProvider || allocation_provider == kWebGpuExecutionProvider) { AllocateMLValue(gpu_alloc, expected_output_dims, &output_ml_value); } else { ORT_THROW("Unsupported provider"); @@ -390,9 +383,9 @@ void RunModelWithBindingMatMul(InferenceSession& session_object, // Now run ASSERT_STATUS_OK(session_object.Run(run_options, *io_binding)); - if ((is_preallocate_output_vec && (allocation_provider == kCudaExecutionProvider || allocation_provider == kRocmExecutionProvider || allocation_provider == kWebGpuExecutionProvider)) || + if ((is_preallocate_output_vec && (allocation_provider == kCudaExecutionProvider || allocation_provider == kWebGpuExecutionProvider)) || (output_device && output_device->Type() == OrtDevice::GPU)) { -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) // in this case we need to copy the tensor from cuda to cpu std::vector& outputs = io_binding->GetOutputs(); ASSERT_EQ(1u, outputs.size()); @@ -403,9 +396,6 @@ void RunModelWithBindingMatMul(InferenceSession& session_object, #ifdef USE_CUDA st = gpu_provider->GetDataTransfer()->CopyTensor(rtensor, cpu_tensor); #endif -#ifdef USE_ROCM - st = GetProviderInfo_ROCM().CreateGPUDataTransfer()->CopyTensor(rtensor, cpu_tensor); -#endif #ifdef USE_WEBGPU st = gpu_provider->GetDataTransfer()->CopyTensor(rtensor, cpu_tensor); #endif @@ -415,7 +405,7 @@ void RunModelWithBindingMatMul(InferenceSession& session_object, VerifyOutputs({ml_value}, expected_output_dims, expected_values_mul_y); #endif } else { - if (allocation_provider == kCudaExecutionProvider || allocation_provider == kRocmExecutionProvider || allocation_provider == kWebGpuExecutionProvider) { + if (allocation_provider == kCudaExecutionProvider || allocation_provider == kWebGpuExecutionProvider) { ASSERT_STATUS_OK(gpu_provider->Sync()); } VerifyOutputs(io_binding->GetOutputs(), expected_output_dims, expected_values_mul_y); @@ -637,9 +627,6 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { InferenceSession session_object(so, GetEnvironment()); #ifdef USE_CUDA ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); -#endif -#ifdef USE_ROCM - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRocmExecutionProvider())); #endif ASSERT_STATUS_OK(session_object.Load(MODEL_URI)); ASSERT_STATUS_OK(session_object.Initialize()); @@ -676,7 +663,7 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { } } -#if (defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING)) || (defined(USE_ROCM) && defined(ENABLE_ROCM_PROFILING)) +#if (defined(USE_CUDA) && defined(ENABLE_CUDA_PROFILING)) ASSERT_TRUE(has_kernel_info); #endif } @@ -692,9 +679,6 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions2) { #ifdef USE_CUDA ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); #endif -#ifdef USE_ROCM - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRocmExecutionProvider())); -#endif #ifdef USE_WEBGPU ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultWebGpuExecutionProvider())); #endif @@ -731,10 +715,6 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions2) { has_api_info = has_api_info || lines[i].find("Api") != std::string::npos && lines[i].find("cudaLaunch") != std::string::npos; #endif -#ifdef USE_ROCM - has_api_info = has_api_info || lines[i].find("Api") != std::string::npos && - lines[i].find("hipLaunch") != std::string::npos; -#endif #ifdef USE_WEBGPU has_api_info = has_api_info || lines[i].find("Api") != std::string::npos; #endif @@ -742,7 +722,7 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions2) { } // Note that the apple device is a paravirtual device which may not support webgpu timestamp query. So skip the check on it. -#if (defined(USE_ROCM) && defined(ENABLE_ROCM_PROFILING)) || (defined(USE_WEBGPU) && !defined(__APPLE__)) +#if (defined(USE_WEBGPU) && !defined(__APPLE__)) ASSERT_TRUE(has_api_info); #endif } @@ -1041,7 +1021,7 @@ static void TestBindHelper(const std::string& log_str, InferenceSession session_object{so, GetEnvironment()}; IExecutionProvider* gpu_provider{}; - if (bind_provider_type == kCudaExecutionProvider || bind_provider_type == kRocmExecutionProvider || bind_provider_type == kWebGpuExecutionProvider) { + if (bind_provider_type == kCudaExecutionProvider || bind_provider_type == kWebGpuExecutionProvider) { #ifdef USE_CUDA { auto provider = DefaultCudaExecutionProvider(); @@ -1049,13 +1029,6 @@ static void TestBindHelper(const std::string& log_str, ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::move(provider))); } #endif -#ifdef USE_ROCM - { - auto provider = DefaultRocmExecutionProvider(); - gpu_provider = provider.get(); - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::move(provider))); - } -#endif #ifdef USE_WEBGPU { ConfigOptions config_options{}; @@ -1176,11 +1149,9 @@ TEST(InferenceSessionTests, InvalidInputTypeOfTensorElement) { ASSERT_TRUE(!st.IsOK()); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #elif USE_WEBGPU constexpr const char* kGpuExecutionProvider = kWebGpuExecutionProvider; #endif @@ -1670,8 +1641,6 @@ TEST(InferenceSessionTests, Test3LayerNestedSubgraph) { ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider())); #elif USE_CUDA ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); -#elif USE_ROCM - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRocmExecutionProvider())); #endif status = session_object.Load(model_file_name); @@ -1822,8 +1791,6 @@ TEST(InferenceSessionTests, Test2LayerNestedSubgraph) { ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultTensorrtExecutionProvider())); #elif USE_CUDA ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultCudaExecutionProvider())); -#elif USE_ROCM - ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(DefaultRocmExecutionProvider())); #endif status = session_object.Load(model_file_name); diff --git a/onnxruntime/test/internal_testing_ep/internal_testing_tests.cc b/onnxruntime/test/internal_testing_ep/internal_testing_tests.cc index ee3824a5ca2f2..74a812062875a 100644 --- a/onnxruntime/test/internal_testing_ep/internal_testing_tests.cc +++ b/onnxruntime/test/internal_testing_ep/internal_testing_tests.cc @@ -161,7 +161,7 @@ TEST(InternalTestingEP, PreventSaveOfModelWithCompiledOps) { // the internal NHWC operators are only included as part of contrib ops currently. as the EP requests the NHWC // version of the ONNX operator when matching a static kernel, those are required. -#if !defined(DISABLE_CONTRIB_OPS) && !defined(USE_ROCM) +#if !defined(DISABLE_CONTRIB_OPS) TEST(InternalTestingEP, TestMixOfStaticAndCompiledKernels) { const ORTCHAR_T* ort_model_path = ORT_MODEL_FOLDER "transform/fusion/conv_relu_opset12.onnx"; diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 9e69156efefa1..8446f88639436 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -56,7 +56,7 @@ void usage() { "\t-v: verbose\n" "\t-n [test_case_name]: Specifies a single test case to run.\n" "\t-e [EXECUTION_PROVIDER]: EXECUTION_PROVIDER could be 'cpu', 'cuda', 'dnnl', 'tensorrt', 'vsinpu'" - "'openvino', 'rocm', 'migraphx', 'acl', 'armnn', 'xnnpack', 'webgpu', 'nnapi', 'qnn', 'snpe' or 'coreml'. " + "'openvino', 'migraphx', 'acl', 'armnn', 'xnnpack', 'webgpu', 'nnapi', 'qnn', 'snpe' or 'coreml'. " "Default: 'cpu'.\n" "\t-p: Pause after launch, can attach debugger and continue\n" "\t-x: Use parallel executor, default (without -x): sequential executor.\n" @@ -228,7 +228,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { bool enable_dml = false; bool enable_acl = false; bool enable_armnn = false; - bool enable_rocm = false; bool enable_migraphx = false; bool enable_webgpu = false; bool enable_xnnpack = false; @@ -319,8 +318,6 @@ int real_main(int argc, char* argv[], Ort::Env& env) { enable_acl = true; } else if (!CompareCString(optarg, ORT_TSTR("armnn"))) { enable_armnn = true; - } else if (!CompareCString(optarg, ORT_TSTR("rocm"))) { - enable_rocm = true; } else if (!CompareCString(optarg, ORT_TSTR("migraphx"))) { enable_migraphx = true; } else if (!CompareCString(optarg, ORT_TSTR("webgpu"))) { @@ -746,17 +743,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); #else fprintf(stderr, "ArmNN is not supported in this build\n"); return -1; -#endif - } - if (enable_rocm) { -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - rocm_options.do_copy_in_default_stream = true; - // TODO: Support arena configuration for users of test runner - sf.AppendExecutionProvider_ROCM(rocm_options); -#else - fprintf(stderr, "ROCM is not supported in this build"); - return -1; #endif } if (enable_migraphx) { diff --git a/onnxruntime/test/optimizer/compute_optimizer_test.cc b/onnxruntime/test/optimizer/compute_optimizer_test.cc index 333c1edf8ffab..08c7a0700030f 100644 --- a/onnxruntime/test/optimizer/compute_optimizer_test.cc +++ b/onnxruntime/test/optimizer/compute_optimizer_test.cc @@ -195,8 +195,6 @@ TEST(ComputeOptimizerTests, GatherND_E2E) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; const std::vector output_names{"output", "gather_output"}; @@ -300,8 +298,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_ScalarSlicingOnBatchDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -406,8 +402,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_SlicingOnBatchDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -504,8 +498,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_ScalarSlicingOnLastDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -602,8 +594,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_SlicingOnLastDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -702,8 +692,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_ScalarSlicingOnSecondLastDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -801,8 +789,6 @@ TEST(ComputeOptimizerTests, GatherMatMul_SlicingOnSecondLastDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1232,8 +1218,6 @@ TEST(ComputeOptimizerTests, GatherReshape_ScalarSlicingOnBatchDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1327,8 +1311,6 @@ TEST(ComputeOptimizerTests, GatherReshape_SlicingOnBatchDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1420,8 +1402,6 @@ TEST(ComputeOptimizerTests, GatherReshape_ScalarSlicingOnSeqlenDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1514,8 +1494,6 @@ TEST(ComputeOptimizerTests, GatherReshape_SlicingOnSeqlenDim) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1608,8 +1586,6 @@ TEST(ComputeOptimizerTests, GatherReshape_SlicingOnSeqlenDim2) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -1781,8 +1757,6 @@ TEST(ComputeOptimizerTests, GatherRobertaE2E) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; @@ -3072,8 +3046,6 @@ TEST(ComputeOptimizerTests, ReshapeMlmBertE2E) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 9f0a2ad2de1c2..70e84733fa869 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -5807,12 +5807,6 @@ TEST_F(GraphTransformationTests, BiasSoftmaxFusionTest_GpuOnly) { tester.TestNoFusionOccurs(); } -TEST_F(GraphTransformationTests, BiasSoftmaxFusionTest_Simple_Rocm) { - constexpr const ORTCHAR_T* model_uri = MODEL_FOLDER "fusion/bias_softmax_fusion_simple.onnx"; - BiasSoftmaxFusionTester tester(model_uri, logger_.get(), kRocmExecutionProvider); - tester.TestFusionOccurs(1, true); -} - TEST_F(GraphTransformationTests, BiasSoftmaxFusionTest_Simple_Cuda) { constexpr const ORTCHAR_T* model_uri = MODEL_FOLDER "fusion/bias_softmax_fusion_simple.onnx"; BiasSoftmaxFusionTester tester(model_uri, logger_.get()); @@ -6515,7 +6509,7 @@ TEST_F(GraphTransformationTests, MatMulScaleFusionWithScaleInput) { }); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST_F(GraphTransformationTests, IsInfReduceSum_Test) { constexpr const ORTCHAR_T* model_uri = MODEL_FOLDER "fusion/isinf_reducesum.onnx"; std::shared_ptr p_model; diff --git a/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc b/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc index adc173456a7db..be9e5bee4df5d 100644 --- a/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc +++ b/onnxruntime/test/optimizer/rule_based_graph_transformer_test.cc @@ -30,7 +30,7 @@ TEST(RuleBasedGraphTransformerTest, TestCompatibleProviders) { Graph& graph = model->MainGraph(); // Create rule based transformer with a dummy rewrite rule and register it with Cuda as compatible provider - InlinedHashSet compatible_provider{onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider}; + InlinedHashSet compatible_provider{onnxruntime::kCudaExecutionProvider}; auto dummy_rule = std::make_unique("DummyRule"); const auto* dummy_rule_ptr = dummy_rule.get(); diff --git a/onnxruntime/test/optimizer/test_optimizer_utils.cc b/onnxruntime/test/optimizer/test_optimizer_utils.cc index 40065c2fc7006..baba334d017fe 100644 --- a/onnxruntime/test/optimizer/test_optimizer_utils.cc +++ b/onnxruntime/test/optimizer/test_optimizer_utils.cc @@ -65,8 +65,6 @@ void RunModelWithData(const PathString& model_uri, const std::string session_log execution_provider = DefaultCpuExecutionProvider(); else if (provider_type == onnxruntime::kCudaExecutionProvider) execution_provider = DefaultCudaExecutionProvider(); - else if (provider_type == onnxruntime::kRocmExecutionProvider) - execution_provider = DefaultRocmExecutionProvider(); EXPECT_TRUE(session_object.RegisterExecutionProvider(std::move(execution_provider)).IsOK()); Status st; diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 2c9377d48f0c4..c27700166e584 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -42,7 +42,7 @@ ABSL_FLAG(std::string, F, "", "[Usage]: -f \"dimension_denotation1:override_value1\" -f \"dimension_denotation2:override_value2\" ... or" " -f \"dimension_denotation1:override_value1 dimension_denotation2 : override_value2... \". Override value must > 0."); ABSL_FLAG(std::string, m, "duration", "Specifies the test mode. Value could be 'duration' or 'times'."); -ABSL_FLAG(std::string, e, "cpu", "Specifies the provider 'cpu','cuda','dnnl','tensorrt', 'nvtensorrtrtx', 'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'rocm', 'migraphx', 'xnnpack', 'vitisai' or 'webgpu'."); +ABSL_FLAG(std::string, e, "cpu", "Specifies the provider 'cpu','cuda','dnnl','tensorrt', 'nvtensorrtrtx', 'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'migraphx', 'xnnpack', 'vitisai' or 'webgpu'."); ABSL_FLAG(size_t, r, DefaultPerformanceTestConfig().run_config.repeated_times, "Specifies the repeated times if running in 'times' test mode."); ABSL_FLAG(size_t, t, DefaultPerformanceTestConfig().run_config.duration_in_seconds, "Specifies the seconds to run for 'duration' mode."); ABSL_FLAG(std::string, p, "", "Specifies the profile name to enable profiling and dump the profile data to the file."); @@ -325,8 +325,6 @@ bool CommandLineParser::ParseArguments(PerformanceTestConfig& test_config, int a test_config.machine_config.provider_type_name = onnxruntime::kAclExecutionProvider; } else if (ep == "armnn") { test_config.machine_config.provider_type_name = onnxruntime::kArmNNExecutionProvider; - } else if (ep == "rocm") { - test_config.machine_config.provider_type_name = onnxruntime::kRocmExecutionProvider; } else if (ep == "migraphx") { test_config.machine_config.provider_type_name = onnxruntime::kMIGraphXExecutionProvider; } else if (ep == "xnnpack") { diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index cb40a9beafeee..3468e2e55c7b6 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -661,16 +661,6 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)"); performance_test_config.run_config.enable_cpu_mem_arena ? 1 : 0)); #else ORT_THROW("ArmNN is not supported in this build\n"); -#endif - } else if (provider_name_ == onnxruntime::kRocmExecutionProvider) { -#ifdef USE_ROCM - OrtROCMProviderOptions rocm_options; - rocm_options.miopen_conv_exhaustive_search = performance_test_config.run_config.cudnn_conv_algo; - rocm_options.do_copy_in_default_stream = !performance_test_config.run_config.do_cuda_copy_in_separate_stream; - // TODO: Support arena configuration for users of perf test - session_options.AppendExecutionProvider_ROCM(rocm_options); -#else - ORT_THROW("ROCM is not supported in this build\n"); #endif } else if (provider_name_ == onnxruntime::kMIGraphXExecutionProvider) { #ifdef USE_MIGRAPHX diff --git a/onnxruntime/test/providers/compare_provider_test_utils.cc b/onnxruntime/test/providers/compare_provider_test_utils.cc index 386a5656d8a01..63120143870d4 100644 --- a/onnxruntime/test/providers/compare_provider_test_utils.cc +++ b/onnxruntime/test/providers/compare_provider_test_utils.cc @@ -32,8 +32,6 @@ std::unique_ptr GetExecutionProvider(const std::string& prov execution_provider = DefaultNnapiExecutionProvider(); else if (provider_type == onnxruntime::kAclExecutionProvider) execution_provider = DefaultAclExecutionProvider(); - else if (provider_type == onnxruntime::kRocmExecutionProvider) - execution_provider = DefaultRocmExecutionProvider(); else if (provider_type == onnxruntime::kDmlExecutionProvider) execution_provider = DefaultDmlExecutionProvider(); else if (provider_type == onnxruntime::kWebGpuExecutionProvider) diff --git a/onnxruntime/test/providers/cpu/activation/activation_op_test.cc b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc index 11a3d67a3e13e..d711e050fb913 100644 --- a/onnxruntime/test/providers/cpu/activation/activation_op_test.cc +++ b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc @@ -172,7 +172,7 @@ TEST_F(ActivationOpTest, Relu) { #endif // MLAS_F16VEC_INTRINSICS_SUPPORTED } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +#if defined(USE_CUDA) || defined(USE_COREML) TEST_F(ActivationOpTest, Sigmoid_fp16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -262,7 +262,7 @@ TEST_F(ActivationOpTest, Relu_fp16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST_F(ActivationOpTest, Sigmoid_bfloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -299,8 +299,6 @@ TEST_F(ActivationOpTest, Sigmoid_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif @@ -339,8 +337,6 @@ TEST_F(ActivationOpTest, Tanh_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif @@ -379,14 +375,12 @@ TEST_F(ActivationOpTest, Relu_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } -#endif // USE_CUDA || USE_ROCM || USE_DNNL +#endif // USE_CUDA || USE_DNNL #if defined(USE_DNNL) TEST_F(ActivationOpTest, LeakyRelu_bfloat16) { diff --git a/onnxruntime/test/providers/cpu/controlflow/if_test.cc b/onnxruntime/test/providers/cpu/controlflow/if_test.cc index 31b5618180bf7..d13371600389f 100644 --- a/onnxruntime/test/providers/cpu/controlflow/if_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/if_test.cc @@ -246,14 +246,11 @@ void RunTest(bool condition_value, excluded_providers.insert(kTensorrtExecutionProvider); } if (options.mixed_execution_providers) { - // we want the GPU (CUDA/ROCm) provider to be first, and the CPU provider second. all except the If should run on + // we want the GPU (CUDA) provider to be first, and the CPU provider second. all except the If should run on // GPU given that, which creates the scenario where we need to copy to/from CPU to execute the If node correctly. std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#endif -#ifdef USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -295,7 +292,7 @@ TEST(If, NoShapeInMainGraph_ShapeInSubgraph_False) { RunTest(false, options, false); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(If, MixedExecutionProviders) { RunOptions options{}; options.mixed_execution_providers = true; @@ -316,7 +313,7 @@ TEST(If, MixedExecutionProvidersNoShapeInSubgraph) { options.include_dim_values_in_subgraph = false; RunTest(true, options); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(If, SymbolicShapeInMainGraph_NoShapeInSubgraph_True) { RunOptions options; diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index 07cd2114372dd..83439960e8ad3 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -360,8 +360,6 @@ void RunTest(int64_t max_iterations, std::vector> execution_providers; #if defined(USE_CUDA) execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -1041,8 +1039,8 @@ TEST(Loop, IterationCountAsOutput) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -#if defined(USE_CUDA) || defined(USE_ROCM) -// test that when part of the subgraph run on CUDA/ROCm it executes successfully +#if defined(USE_CUDA) +// test that when part of the subgraph run on CUDA it executes successfully TEST(Loop, MixedExecutionProviders) { RunOptions options{}; options.mixed_execution_providers = true; diff --git a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc index 6bf2fc63ab165..c7de8d4cba83d 100644 --- a/onnxruntime/test/providers/cpu/controlflow/scan_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/scan_test.cc @@ -412,8 +412,6 @@ static void RunTest_v9(const std::string test_name, int64_t sequence_len, int64_ std::vector> execution_providers; #if defined(USE_CUDA) execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -1167,8 +1165,6 @@ void UnknownDimInSubgraphOutput(bool is_v8, bool mixed_execution_providers = fal std::vector> execution_providers; #if defined(USE_CUDA) execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif defined(USE_ROCM) - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif execution_providers.push_back(DefaultCpuExecutionProvider()); @@ -1181,7 +1177,7 @@ void UnknownDimInSubgraphOutput(bool is_v8, bool mixed_execution_providers = fal TEST_8_AND_9(UnknownDimInSubgraphOutput); -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(Scan, MixedExecutionProviders) { RunOptions options{}; options.is_v8 = false; diff --git a/onnxruntime/test/providers/cpu/generator/random_test.cc b/onnxruntime/test/providers/cpu/generator/random_test.cc index a923df2cebe30..b44aff56f1153 100644 --- a/onnxruntime/test/providers/cpu/generator/random_test.cc +++ b/onnxruntime/test/providers/cpu/generator/random_test.cc @@ -37,7 +37,7 @@ TEST(Random, RandomNormal2DDouble) { // The expected_output is generated using std lib, which is used by CPU kernel only. // So we need to exclude other EPs here. Ditto for other places. test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider}); } void RunRandomNormalLike3DFloat(bool infer_dtype = false) { @@ -74,7 +74,7 @@ void RunRandomNormalLike3DFloat(bool infer_dtype = false) { // TensorRT does not support manual seed overrides and there will be result mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(Random, RandomNormalLike3DDouble) { @@ -112,7 +112,7 @@ TEST(Random, RandomUniform1DFloat) { // TensorRT does not support manual seed overrides and there will be result mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } void RunRandomUniformLikeTest(bool infer_dtype = false) { @@ -146,7 +146,7 @@ void RunRandomUniformLikeTest(bool infer_dtype = false) { // TensorRT does not support seed parameter and there will be result mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kTensorrtExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(Random, RandomUniformLike2DDouble) { @@ -333,7 +333,7 @@ TEST(Random, MultinomialInvalidDtype) { test.Run(OpTester::ExpectResult::kExpectFailure, "Output type must be int32 or int64"); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) // We cannot call CUDA lib from UT, so just do some simple verification on output tensor. void RunRandomNormalGpuTest(const std::vector dims, const float mean, const float scale, const float seed, TensorProto_DataType dtype, bool is_random_like, bool infer_dtype) { diff --git a/onnxruntime/test/providers/cpu/math/einsum_test.cc b/onnxruntime/test/providers/cpu/math/einsum_test.cc index f9cbe46944d66..d3ea8552f60f4 100644 --- a/onnxruntime/test/providers/cpu/math/einsum_test.cc +++ b/onnxruntime/test/providers/cpu/math/einsum_test.cc @@ -380,17 +380,6 @@ TEST(Einsum, ExplicitEinsumAsDiagonalOpWithTranspose) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", ExcludeTrtOnA100()); } -// ROCm doesn't support double -#ifndef USE_ROCM -TEST(Einsum, ExplicitEinsumAsDiagonalOpWithTranspose_double) { - OpTester test("Einsum", 12, onnxruntime::kOnnxDomain); - test.AddAttribute("equation", "iji->ji"); - test.AddInput("x", {2, 2, 2}, {1., 2., 3., 4., 1., 2., 3., 4.}); - test.AddOutput("o", {2, 2}, {1., 2., 3., 4.}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", ExcludeTrtOnA100()); -} -#endif - TEST(Einsum, ExplicitEinsumAsDiagonalOpWithTranspose_int32) { OpTester test("Einsum", 12, onnxruntime::kOnnxDomain); test.AddAttribute("equation", "iji->ji"); diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index cbb8ca43e8f06..3fb8cc3e1544f 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -40,8 +40,6 @@ void TestBinaryFloat16(const char* op_name, execution_providers.push_back(DefaultCoreMLExecutionProvider(true)); #elif USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif if (execution_providers.size() > 0) { OpTester tester(op_name, 14); @@ -56,8 +54,6 @@ void TestBinaryFloat16(const char* op_name, std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif if (enable_bf16 && execution_providers.size() > 0) { @@ -84,8 +80,6 @@ void TestUnaryFloat16(const char* op_name, execution_providers.push_back(DefaultCoreMLExecutionProvider(true)); #elif USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif if (execution_providers.size() > 0) { OpTester tester(op_name, opset); @@ -100,8 +94,6 @@ void TestUnaryFloat16(const char* op_name, std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif if (run_bf16 && execution_providers.size() > 0) { @@ -1439,7 +1431,7 @@ TEST(MathOpTest, Pow_float16_float16) { dims, {1.0f, 256.0f, 2.0f, 1.0f}, false); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +#if defined(USE_CUDA) || defined(USE_COREML) TEST(MathOpTest, Pow_float_float16) { OpTester test("Pow", 12); std::vector dims{4}; @@ -1451,8 +1443,6 @@ TEST(MathOpTest, Pow_float_float16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_COREML execution_providers.push_back(DefaultCoreMLExecutionProvider(true)); #endif @@ -4079,19 +4069,17 @@ TEST(ModOpTest, Fmod_float16_mixed_sign) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kQnnExecutionProvider}); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(ModOpTest, Fmod_bfloat16_mixed_sign) { OpTester test("Mod", 13); test.AddAttribute("fmod", 1); - // Due to BFloat16's precision, if the result is too small, it's not easy get pass for both CUDA and ROCm. + // Due to BFloat16's precision, if the result is too small, it's not easy get pass for both CUDA. test.AddInput("X", {4}, MakeBFloat16({8.0f, 5.0f, -8.0f, 8.0f})); test.AddInput("Y", {4}, MakeBFloat16({-3.4f, 8.0f, 3.4f, 5.0f})); test.AddOutput("Z", {4}, MakeBFloat16({1.2f, 5.f, -1.2f, 3.f})); std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/onnxruntime/test/providers/cpu/math/gemm_test.cc b/onnxruntime/test/providers/cpu/math/gemm_test.cc index 0e5a4dac465b1..d7d9d2994afa1 100644 --- a/onnxruntime/test/providers/cpu/math/gemm_test.cc +++ b/onnxruntime/test/providers/cpu/math/gemm_test.cc @@ -95,7 +95,7 @@ auto get_bias_value = [](const std::vector& bias_data, BiasType bias_type } // namespace -// Only CUDA, ROCM, CoreML and XNNPack kernels have float 16 support +// Only CUDA, CoreML and XNNPack kernels have float 16 support TEST(GemmOpTest, GemmNoTrans_f16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -195,7 +195,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) { } } -// Only CUDA, ROCM and CoreML kernels have float 16 support +// Only CUDA and CoreML kernels have float 16 support TEST(GemmOpTest, GemmTransB_f16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -242,7 +242,7 @@ TEST(GemmOpTest, GemmTransB_f16) { } } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST(GemmOpTest, GemmNoTrans_bfloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -270,13 +270,6 @@ TEST(GemmOpTest, GemmNoTrans_bfloat16) { test.Config(run_with_tunable_op); #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/true)); - test.ConfigEps(std::move(execution_providers)) - .RunWithConfig(); - - execution_providers.clear(); - execution_providers.emplace_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/false)); #elif USE_DNNL execution_providers.emplace_back(DefaultDnnlExecutionProvider()); #endif diff --git a/onnxruntime/test/providers/cpu/math/matmul_test.cc b/onnxruntime/test/providers/cpu/math/matmul_test.cc index b7f2b5800560a..2e56aa6767598 100644 --- a/onnxruntime/test/providers/cpu/math/matmul_test.cc +++ b/onnxruntime/test/providers/cpu/math/matmul_test.cc @@ -365,7 +365,7 @@ TEST(MathOpTest, MatMulFloatType) { RunMatMulTest(7, false, true); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) || defined(USE_XNNPACK) +#if defined(USE_CUDA) || defined(USE_COREML) || defined(USE_XNNPACK) TEST(MathOpTest, MatMulFloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -445,7 +445,7 @@ TEST(MathOpTest, MatMulZeroKInt32Type) { RunMatMulZeroKTest(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) || defined(USE_XNNPACK) +#if defined(USE_CUDA) || defined(USE_COREML) || defined(USE_XNNPACK) TEST(MathOpTest, MatMul_Float16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -482,7 +482,7 @@ TEST(MathOpTest, MatMul_Float16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST(MathOpTest, MatMul_bfloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -506,13 +506,6 @@ TEST(MathOpTest, MatMul_bfloat16) { test.Config(run_with_tunable_op); #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/true)); - test.ConfigEps(std::move(execution_providers)) - .RunWithConfig(); - - execution_providers.clear(); - execution_providers.emplace_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/false)); #elif USE_DNNL execution_providers.emplace_back(DefaultDnnlExecutionProvider()); #endif diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc index 215203b31f49c..962a055b5fcbe 100644 --- a/onnxruntime/test/providers/cpu/math/softmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -66,7 +66,7 @@ TEST(SoftmaxOperator, webgpu_nan) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_XNNPACK) +#if defined(USE_CUDA) || defined(USE_XNNPACK) TEST(SoftmaxOperator, Simple_fp16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -95,7 +95,7 @@ TEST(SoftmaxOperator, Simple_fp16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST(SoftmaxOperator, Simple_bfloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -120,14 +120,12 @@ TEST(SoftmaxOperator, Simple_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } -#endif // USE_CUDA USE_ROCM USE_DNNL +#endif // USE_CUDA USE_DNNL TEST(SoftmaxOperator, LargeNumber) { // x = np.array([[0, 1, 2, 3], [10000, 10001, 10002, 10003]]).astype(np.float32) diff --git a/onnxruntime/test/providers/cpu/model_tests.cc b/onnxruntime/test/providers/cpu/model_tests.cc index ca1a3104e0bed..b1642161d0bb8 100644 --- a/onnxruntime/test/providers/cpu/model_tests.cc +++ b/onnxruntime/test/providers/cpu/model_tests.cc @@ -238,7 +238,7 @@ TEST_P(ModelTest, Run) { // when cuda or openvino is enabled, set it to a larger value for resolving random MNIST test failure if (model_path.find(ORT_TSTR("_MNIST")) > 0) { - if (provider_name == "cuda" || provider_name == "openvino" || provider_name == "rocm") { + if (provider_name == "cuda" || provider_name == "openvino") { per_sample_tolerance = 2.5e-2; relative_per_sample_tolerance = 1e-2; } @@ -331,9 +331,6 @@ TEST_P(ModelTest, Run) { cuda_options.Update(options); ortso.AppendExecutionProvider_CUDA_V2(*cuda_options); - } else if (provider_name == "rocm") { - OrtROCMProviderOptions ep_options; - ortso.AppendExecutionProvider_ROCM(ep_options); } #ifdef USE_DNNL else if (provider_name == "dnnl") { @@ -545,7 +542,6 @@ static constexpr ORT_STRING_VIEW provider_name_migraphx = ORT_TSTR("migraphx"); #endif static constexpr ORT_STRING_VIEW provider_name_openvino = ORT_TSTR("openvino"); static constexpr ORT_STRING_VIEW provider_name_cuda = ORT_TSTR("cuda"); -static constexpr ORT_STRING_VIEW provider_name_rocm = ORT_TSTR("rocm"); static constexpr ORT_STRING_VIEW provider_name_dnnl = ORT_TSTR("dnnl"); // For any non-Android system, NNAPI will only be used for ort model converter #if defined(USE_NNAPI) && defined(__ANDROID__) @@ -588,9 +584,6 @@ ::std::vector<::std::basic_string> GetParameterStrings() { #ifdef USE_CUDA provider_names[provider_name_cuda] = {opset7, opset8, opset9, opset10, opset11, opset12, opset13, opset14, opset15, opset16, opset17, opset18}; #endif -#ifdef USE_ROCM - provider_names[provider_name_rocm] = {opset7, opset8, opset9, opset10, opset11, opset12, opset13, opset14, opset15, opset16, opset17, opset18}; -#endif #ifdef USE_DNNL provider_names[provider_name_dnnl] = {opset10}; #endif @@ -663,46 +656,29 @@ ::std::vector<::std::basic_string> GetParameterStrings() { ORT_TSTR("operator_pow"), }; - static const ORTCHAR_T* cuda_rocm_flaky_tests[] = {ORT_TSTR("fp16_inception_v1"), - ORT_TSTR("fp16_shufflenet"), - ORT_TSTR("fp16_tiny_yolov2"), - ORT_TSTR("candy"), - ORT_TSTR("tinyyolov3"), - ORT_TSTR("mlperf_ssd_mobilenet_300"), - ORT_TSTR("mlperf_ssd_resnet34_1200"), - ORT_TSTR("tf_inception_v1"), - ORT_TSTR("faster_rcnn"), - ORT_TSTR("split_zero_size_splits"), - ORT_TSTR("convtranspose_3d"), - ORT_TSTR("fp16_test_tiny_yolov2-Candy"), - ORT_TSTR("fp16_coreml_FNS-Candy"), - ORT_TSTR("fp16_test_tiny_yolov2"), - ORT_TSTR("fp16_test_shufflenet"), - ORT_TSTR("keras2coreml_SimpleRNN_ImageNet"), - // models from model zoo. #26274: cuDNN frontend no valid engine - ORT_TSTR("YOLOv3"), - ORT_TSTR("YOLOv3-12"), - ORT_TSTR("YOLOv4"), - ORT_TSTR("SSD-MobilenetV1"), - ORT_TSTR("SSD-MobilenetV1-12")}; - - // For ROCm EP, also disable the following tests due to flakiness, - // mainly with precision issue and random memory access fault. - static const ORTCHAR_T* rocm_disabled_tests[] = {ORT_TSTR("bvlc_alexnet"), - ORT_TSTR("bvlc_reference_caffenet"), - ORT_TSTR("bvlc_reference_rcnn_ilsvrc13"), - ORT_TSTR("coreml_Resnet50_ImageNet"), - ORT_TSTR("mlperf_resnet"), - ORT_TSTR("mobilenetv2-1.0"), - ORT_TSTR("shufflenet"), - // models from model zoo - ORT_TSTR("AlexNet"), - ORT_TSTR("CaffeNet"), - ORT_TSTR("MobileNet v2-7"), - ORT_TSTR("R-CNN ILSVRC13"), - ORT_TSTR("ShuffleNet-v1"), - ORT_TSTR("version-RFB-320"), - ORT_TSTR("version-RFB-640")}; + static const ORTCHAR_T* cuda_flaky_tests[] = {ORT_TSTR("fp16_inception_v1"), + ORT_TSTR("fp16_shufflenet"), + ORT_TSTR("fp16_tiny_yolov2"), + ORT_TSTR("candy"), + ORT_TSTR("tinyyolov3"), + ORT_TSTR("mlperf_ssd_mobilenet_300"), + ORT_TSTR("mlperf_ssd_resnet34_1200"), + ORT_TSTR("tf_inception_v1"), + ORT_TSTR("faster_rcnn"), + ORT_TSTR("split_zero_size_splits"), + ORT_TSTR("convtranspose_3d"), + ORT_TSTR("fp16_test_tiny_yolov2-Candy"), + ORT_TSTR("fp16_coreml_FNS-Candy"), + ORT_TSTR("fp16_test_tiny_yolov2"), + ORT_TSTR("fp16_test_shufflenet"), + ORT_TSTR("keras2coreml_SimpleRNN_ImageNet"), + // models from model zoo. #26274: cuDNN frontend no valid engine + ORT_TSTR("YOLOv3"), + ORT_TSTR("YOLOv3-12"), + ORT_TSTR("YOLOv4"), + ORT_TSTR("SSD-MobilenetV1"), + ORT_TSTR("SSD-MobilenetV1-12")}; + static const ORTCHAR_T* openvino_disabled_tests[] = { ORT_TSTR("tf_mobilenet_v1_1.0_224"), ORT_TSTR("bertsquad"), @@ -827,13 +803,9 @@ ::std::vector<::std::basic_string> GetParameterStrings() { std::unordered_set> all_disabled_tests(std::begin(immutable_broken_tests), std::end(immutable_broken_tests)); - bool provider_cuda_or_rocm = provider_name == provider_name_cuda; - if (provider_name == provider_name_rocm) { - provider_cuda_or_rocm = true; - all_disabled_tests.insert(std::begin(rocm_disabled_tests), std::end(rocm_disabled_tests)); - } - if (provider_cuda_or_rocm) { - all_disabled_tests.insert(std::begin(cuda_rocm_flaky_tests), std::end(cuda_rocm_flaky_tests)); + bool provider_cuda = provider_name == provider_name_cuda; + if (provider_cuda) { + all_disabled_tests.insert(std::begin(cuda_flaky_tests), std::end(cuda_flaky_tests)); } else if (provider_name == provider_name_dml) { all_disabled_tests.insert(std::begin(dml_disabled_tests), std::end(dml_disabled_tests)); } else if (provider_name == provider_name_dnnl) { diff --git a/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc index a529d572d7cca..93ca22a16bf67 100644 --- a/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc @@ -703,8 +703,8 @@ TEST(BatchNormTest, NonSpatial_Complicated) { 8); // opset-8 } -// Only CUDA and ROCm kernels have float 16 support -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +// Only CUDA/CoreML kernels have float 16 support +#if defined(USE_CUDA) || defined(USE_COREML) TEST(BatchNormTest, BatchNorm2d_fp16) { vector X{-0.91221f, -0.283559f, 0.937637f, 2.09818f, -0.100199f, -0.608113f, 0.444562f, -1.07505f, 0.940591f, -0.922262f, 0.0931303f, 0.69611f, 1.55187f, 0.159808f, 0.914874f, -1.24856f, -1.98928f, -0.331621f, @@ -923,7 +923,7 @@ TEST(BatchNormTest, ForwardTrainingTestWithSavedOutputsOpset9) { // exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm() test.Run(OpTester::ExpectResult::kExpectSuccess, "", // TODO(mtavenrath) flakiness of running_mean for CUDA has been fixed, the delta of running_var is still ~0.1 - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider, kWebGpuExecutionProvider}); } @@ -953,7 +953,7 @@ TEST(BatchNormTest, ForwardTrainingTestOpset14) { // exclude CUDA Execution Provider due to flakiness // exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm() test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider, kWebGpuExecutionProvider}); } @@ -983,7 +983,7 @@ TEST(BatchNormTest, ForwardTrainingTestOpset15) { // Same exclusions as the opset 14 test test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider, kDnnlExecutionProvider, kWebGpuExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc index 9e0516fd394ce..86ecee5be92dd 100644 --- a/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/instance_norm_op_test.cc @@ -130,8 +130,8 @@ TEST(InstanceNormalizationOpTest, InstanceNormBatch2) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } -// Only CUDA and ROCm kernels have float 16 support -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) || defined(USE_WEBGPU) +// Only a few EPs have float 16 support +#if defined(USE_CUDA) || defined(USE_COREML) || defined(USE_WEBGPU) TEST(InstanceNormalizationOpTest, InstanceNormBatch1_fp16) { OpTester test("InstanceNormalization"); diff --git a/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc index b2b7f1701107a..9be733e22f2e6 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc @@ -166,7 +166,7 @@ TEST(PoolFp16Test, MaxPool_DilationPadding_1d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(PoolFp16Test, MaxPool_Dilation_2d) { @@ -223,7 +223,7 @@ TEST(PoolFp16Test, MaxPool_DilationPadding_2d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(PoolFp16Test, MaxPool_Dilation_Ceil0_2d) { @@ -319,7 +319,7 @@ TEST(PoolTest, MaxPool_DilationPadding_3d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(PoolBF16Test, AveragePool) { diff --git a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc index 1df640a84a64d..8d276b7300e37 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc @@ -474,7 +474,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_1d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(PoolTest, MaxPool_10_Dilation_2d) { @@ -558,7 +558,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_2d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(PoolTest, MaxPool_10_Dilation_Ceil0_2d) { @@ -683,7 +683,7 @@ TEST(PoolTest, MaxPool_10_DilationPadding_3d) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TYPED_TEST(PoolTest, GlobalMaxPool) { diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index c56aa3fb5feac..2ddb9d32cf196 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -881,7 +881,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_float_no_reduction_keepdims) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(ReductionOpTest, ReduceLogSumExp_half) { OpTester test("ReduceLogSumExp"); test.AddAttribute("axes", std::vector{0, 2}); @@ -898,7 +898,7 @@ TEST(ReductionOpTest, ReduceLogSumExp_half) { test.AddOutput("reduced", {1, 2, 1}, FloatsToMLFloat16s({10.33174133f, 12.33174133f})); test.Run(); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(ReductionOpTest, ReduceLogSumExp_int32) { OpTester test("ReduceLogSumExp"); @@ -1375,7 +1375,7 @@ TEST(ReductionOpTest, ReduceMax_double) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +#if defined(USE_CUDA) || defined(USE_COREML) TEST(ReductionOpTest, ReduceMax_half) { OpTester test("ReduceMax"); test.AddAttribute("axes", std::vector{1, 2}); @@ -1392,7 +1392,7 @@ TEST(ReductionOpTest, ReduceMax_half) { test.AddOutput("reduced", {3, 1, 1}, FloatsToMLFloat16s({4.0f, 8.0f, 12.0f})); test.Run(); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(ReductionOpTest, ReduceMax_int32) { OpTester test("ReduceMax"); @@ -2158,7 +2158,7 @@ TEST(ReductionOpTest, ReduceMin_double) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +#if defined(USE_CUDA) || defined(USE_COREML) TEST(ReductionOpTest, ReduceMin_half) { OpTester test("ReduceMin"); test.AddAttribute("axes", std::vector{0, 2}); @@ -2175,7 +2175,7 @@ TEST(ReductionOpTest, ReduceMin_half) { test.AddOutput("reduced", {1, 2, 1}, FloatsToMLFloat16s({1.0f, 3.0f})); test.Run(); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(ReductionOpTest, ReduceMin_int32) { OpTester test("ReduceMin"); @@ -2356,7 +2356,7 @@ TEST(ReductionOpTest, ReduceSum_int32) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_COREML) +#if defined(USE_CUDA) || defined(USE_COREML) TEST(ReductionOpTest, ReduceSumHalfHalf) { OpTester test("ReduceSum"); test.AddAttribute("keepdims", (int64_t)0); @@ -2448,7 +2448,7 @@ TEST(ReductionOpTest, ReduceSum_half_bert) { // Add more UTs for half as needed #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DNNL) +#if defined(USE_CUDA) || defined(USE_DNNL) TEST(ReductionOpTest, ReduceSum_bfloat16) { #ifdef USE_DNNL if (!DnnlHasBF16Support()) { @@ -2465,19 +2465,15 @@ TEST(ReductionOpTest, ReduceSum_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #elif USE_DNNL execution_providers.push_back(DefaultDnnlExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } -#endif // USE_CUDA USE_ROCM USE_DNNL +#endif // USE_CUDA USE_DNNL // on CUDA - this UT, with axes {0,2}, will go thru cudnn lib only if ATenOp is not initialized -// on ROCM - miopen call succeeded, but results in data error, thus follow the same logic done in cudnn for now -// 4.2 doesn't run properly (data error), thus enable the UT only above 4.3 -#if defined(USE_CUDA) || (defined(USE_ROCM) && ROCM_VERSION >= 40300) +#if defined(USE_CUDA) TEST(ReductionOpTest, ReduceSumBFloat16_2) { OpTester test("ReduceSum", 14); test.AddAttribute("keepdims", (int64_t)0); @@ -2488,8 +2484,6 @@ TEST(ReductionOpTest, ReduceSumBFloat16_2) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -2595,7 +2589,7 @@ TEST(ReductionOpTest, ReduceSum_batch_by_seq_by_128) { } } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(ReductionOpTest, ReduceSum_batch_by_seq_by_30528) { test_apex_reduce_sum(4 * 128, 30528); test_apex_reduce_sum(4 * 512, 30528); @@ -3783,7 +3777,7 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero1b) { // test that PrepareForReduce handles this case. Called by all reduction ops so any op can be used in the test TEST(ReductionOpTest, ReduceDimWithZero1) { // TODO: Unskip when fixed #41968513 - if (DefaultDmlExecutionProvider().get() != nullptr || DefaultRocmExecutionProvider().get() != nullptr) { + if (DefaultDmlExecutionProvider().get() != nullptr) { GTEST_SKIP() << "Skipping because of the following error: Expected output shape [{1,0,1}] did not match run output shape [{1,1,1}] for reduced"; } @@ -3834,7 +3828,7 @@ TEST(ReductionOpTest, OptimizeShapeForFastReduce_ReduceDimWithZero2) { TEST(ReductionOpTest, ReduceDimWithZero2) { // TODO: Unskip when fixed #41968513 - if (DefaultDmlExecutionProvider().get() != nullptr || DefaultRocmExecutionProvider().get() != nullptr) { + if (DefaultDmlExecutionProvider().get() != nullptr) { GTEST_SKIP() << "Skipping because of the following error: Can't reduce on dim with value of 0 if 'keepdims' is false. Invalid output shape would be produced. input_shape:{?,0,?}"; } @@ -6046,7 +6040,6 @@ void test_empty_set(const std::string& op, int opset, bool axes_as_input, float kMIGraphXExecutionProvider, kOpenVINOExecutionProvider, kQnnExecutionProvider, - kRocmExecutionProvider, kTensorrtExecutionProvider, kWebGpuExecutionProvider, }); diff --git a/onnxruntime/test/providers/cpu/tensor/expand_test.cc b/onnxruntime/test/providers/cpu/tensor/expand_test.cc index 38e3bc3af6d6b..1680f21d781b7 100644 --- a/onnxruntime/test/providers/cpu/tensor/expand_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/expand_test.cc @@ -5,7 +5,7 @@ #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) #include "test/providers/kernel_compute_test_utils.h" #endif @@ -201,12 +201,10 @@ TEST(ExpandOpTest, Expand_scalar_int32) { test.Run(); } -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) TEST(ExpandOpTest, Strided) { #ifdef USE_CUDA const char* provider = kCudaExecutionProvider; -#else // USE_ROCM - const char* provider = kRocmExecutionProvider; #endif // Generate contiguous output. { diff --git a/onnxruntime/test/providers/cpu/tensor/gather_elements_op_test.cc b/onnxruntime/test/providers/cpu/tensor/gather_elements_op_test.cc index 81e51375b9992..23b4424b1453a 100644 --- a/onnxruntime/test/providers/cpu/tensor/gather_elements_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/gather_elements_op_test.cc @@ -9,7 +9,7 @@ #include "test/providers/provider_test_utils.h" #include "test/util/include/default_providers.h" -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) #include "test/providers/kernel_compute_test_utils.h" #endif @@ -216,7 +216,7 @@ void RunTestWrapper() { test8.Run(); } -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) template void RunKernelComputeTest(std::initializer_list input_dims, std::initializer_list indices_dims, std::initializer_list indices_strides = {}, bool has_axis = false, @@ -228,8 +228,6 @@ void RunKernelComputeTest(std::initializer_list input_dims, std::initia GetData(input_dims, indices_dims, indices_strides, new_axis, input_data, indices_data, output_data); #ifdef USE_CUDA const char* provider = kCudaExecutionProvider; -#else // USE_ROCM - const char* provider = kRocmExecutionProvider; #endif KernelComputeTester test("GatherElements", provider); if (has_axis) test.AddAttribute("axis", axis); @@ -391,7 +389,7 @@ TEST(GatherElementsOpTest, IndicesOutOfBounds) { // skip QNN because it doesn't support out of bounds indices // skip WebGPU because it doesn't support out of bounds indices test.Run(OpTester::ExpectResult::kExpectFailure, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kOpenVINOExecutionProvider, + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kOpenVINOExecutionProvider, kTensorrtExecutionProvider, kDmlExecutionProvider, kQnnExecutionProvider, kWebGpuExecutionProvider}); } @@ -413,7 +411,7 @@ TEST(GatherElementsOpTest, BigIndices) { test1.Run(); } -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) TEST(GatherElementsOpTest, Strided_float) { RunKernelComputeTestWrapper(); } TEST(GatherElementsOpTest, Strided_double) { RunKernelComputeTestWrapper(); } diff --git a/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc b/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc index be79a6d29d539..997ff2869592c 100644 --- a/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/gather_op_test.cc @@ -107,7 +107,7 @@ TEST(GatherOpTest, Gather_invalid_index_cpu) { .RunWithConfig(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(GatherOpTest, Gather_invalid_index_gpu) { OpTester test("Gather"); // Invalid index 3. data[3] does not exist. @@ -126,8 +126,6 @@ TEST(GatherOpTest, Gather_invalid_index_gpu) { test #if defined(USE_CUDA) .ConfigEp(DefaultCudaExecutionProvider()) -#else - .ConfigEp(DefaultRocmExecutionProvider()) #endif .RunWithConfig(); } @@ -440,9 +438,6 @@ TEST(ShrunkenGatherOpTest, ShrunkenGather_PositiveAxis) { #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); #endif -#ifdef USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); -#endif OpTester test("ShrunkenGather", 1, onnxruntime::kMSDomain); test.AddAttribute("axis", 0LL); @@ -464,9 +459,6 @@ TEST(ShrunkenGatherOpTest, ShrunkenGather_NegativeAxis) { #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); #endif -#ifdef USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); -#endif OpTester test("ShrunkenGather", 1, onnxruntime::kMSDomain); test.AddAttribute("axis", -1LL); @@ -488,9 +480,6 @@ TEST(ShrunkenGatherOpTest, ShrunkenGather_InvalidIndicesRank) { #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); #endif -#ifdef USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); -#endif OpTester test("ShrunkenGather", 1, onnxruntime::kMSDomain); test.AddAttribute("axis", 0LL); @@ -512,9 +501,6 @@ TEST(ShrunkenGatherOpTest, ShrunkenGather_InvalidInputRank) { #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); #endif -#ifdef USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); -#endif OpTester test("ShrunkenGather", 1, onnxruntime::kMSDomain); test.AddAttribute("axis", 0LL); diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index 46acb5a730a78..bd8aad5f85514 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -362,8 +362,8 @@ TEST(DequantizeLinearOpTest, Per_Channel_Axis_1_int32) { 0, 4, 16, 48, 0, 20, 80, 240}); // Disable Tensorrt EP due to error, only activation types allowed as input to this layer. - // Disable CUDA, ROCm EP, there is no implementation for int32_t. - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kCudaExecutionProvider, kRocmExecutionProvider}); + // Disable CUDA EP, there is no implementation for int32_t. + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kCudaExecutionProvider}); } // 1d zero & scale with uint8 broadcast axis -2 (-2 resolves to axis 0) diff --git a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc index bb053bc37ce30..be3516437b1aa 100644 --- a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc @@ -109,9 +109,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extr test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA | WEBGPU: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider, kWebGpuExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kWebGpuExecutionProvider}); } TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extrapolation_uint8) { @@ -140,9 +139,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extr test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider}); } TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_with_extrapolation_int8) { @@ -198,11 +196,10 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_without_e test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch // DML: results mismatch test.Run( OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kDmlExecutionProvider}); } TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_tf_crop_and_resize_without_extrapolation_int8) { @@ -283,9 +280,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear) { test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA | WEBGPU: result mismatch due to not implementing NHWC support - // ROCm: results mismatch // TRT: Segmentation fault in A100 - std::unordered_set excluded_providers({kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kWebGpuExecutionProvider}); + std::unordered_set excluded_providers({kCudaExecutionProvider, kCudaNHWCExecutionProvider, kWebGpuExecutionProvider}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", ExcludeTrtOnA100(excluded_providers)); } @@ -309,9 +305,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_uint8) { test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider}); } TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_int8) { @@ -549,9 +544,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_align_corners_uin test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider}); }; run_test(false); @@ -648,10 +642,9 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixe test.AddOutput("Y", {N, sizes[1], sizes[2], C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch // DML: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, kDmlExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kDmlExecutionProvider}); } TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixel_int8) { @@ -761,9 +754,8 @@ TEST(ResizeOpTest, NhwcResizeOpLinearUpSampleTest_4DBilinear_asymmetric_uint8) { test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y, false, .0f, 1.0f); // CUDA: result mismatch due to not implementing NHWC support - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider}); }; run_test(false); @@ -2237,12 +2229,12 @@ TEST(ResizeOpTest, Antialias_NhwcBilinear) { 36.590908f, 76.59091f, 116.59091f}; // Nchw is not supported by CUDA Resize implementation - InlinedVector excluded_eps = {kCudaExecutionProvider, kRocmExecutionProvider}; + InlinedVector excluded_eps = {kCudaExecutionProvider}; TestAntialiasing({{"mode", "linear"}, {"exclude_outside", "1"}}, {1, 5, 8, 3}, X, {1, 4, 5, 3}, Y, excluded_eps); } TEST(ResizeOpTest, Antialias_NhwcBilinear_dtype) { - InlinedVector excluded_eps = {kCudaExecutionProvider, kRocmExecutionProvider}; + InlinedVector excluded_eps = {kCudaExecutionProvider}; { std::vector X(16); std::iota(X.begin(), X.end(), uint8_t(0)); @@ -2389,7 +2381,7 @@ TEST(ResizeOpTest, Antialias_NHWCBicubic_ExcludeOutside) { 46.606194f, 19.878183f, 43.87818f, 21.358122f, 45.35812f, 22.907503f, 46.907505f, 24.387442f, 48.387444f}; - InlinedVector excluded_eps = {kCudaExecutionProvider, kRocmExecutionProvider}; + InlinedVector excluded_eps = {kCudaExecutionProvider}; TestAntialiasing({{"mode", "cubic"}, {"exclude_outside", "0"}}, {1, 4, 6, 2}, X, {1, 8, 4, 2}, Y, excluded_eps); } @@ -2485,7 +2477,7 @@ TEST(ResizeOpTest, NoAntialias_AlignCorners_Cubic_Floor_NHWC) { 23.0000f, 24.0000f, }; // clang-format on - InlinedVector excluded_eps = {kCudaExecutionProvider, kRocmExecutionProvider}; + InlinedVector excluded_eps = {kCudaExecutionProvider}; TestAntialiasing( {{"antialias", "0"}, {"coordinate_transformation_mode", "align_corners"}, @@ -2517,7 +2509,7 @@ TEST(ResizeOpTest, Antialias_Linear_AlignCorners) { 187.08333f, 195.91667f, 198.41667f, 205.91667f, 208.41667f, 217.25f, 219.75f, 227.25f, 229.75f, 238.58333f, 241.08333f, 248.58333f, 251.08333f}; - InlinedVector excluded_eps = {kCudaExecutionProvider, kRocmExecutionProvider}; + InlinedVector excluded_eps = {kCudaExecutionProvider}; TestAntialiasing( {{"mode", "linear"}, {"exclude_outside", "0"}, {"coordinate_transformation_mode", "align_corners"}}, {4, 1, 4, 4, 4}, X, {4, 1, 3, 2, 2}, Y, excluded_eps); diff --git a/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc b/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc index 479a515403c74..56856211c39b3 100644 --- a/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/scatter_op_test.cc @@ -268,7 +268,7 @@ static void scatter_invalid_index(const char* op_name, int op_version) { test.AddOutput("y", {4, 2, 1}, {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 5.0f, 0.0f}); test.Run(OpTester::ExpectResult::kExpectFailure, "indices element out of data bounds, idx=4 must be within the inclusive range [-4,3]", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider, kQnnExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kQnnExecutionProvider}); } TEST(Scatter, InvalidIndex) { diff --git a/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc index 688b2cd39c8fb..3e50b23353cb8 100644 --- a/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/tile_op_test.cc @@ -152,7 +152,7 @@ void RunTestWrapper() { RunTest({}, {}); #endif -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) // _TileMemcpyKernelFromInput, vectorized 4 RunTest({256, 512}, {3, 1}); @@ -263,7 +263,7 @@ TEST(TensorOpTest, TileStringType) { RunTestWrapper(); } TEST(TensorOpTest, TileBoolType) { RunTestWrapperForBool(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_WEBGPU) +#if defined(USE_CUDA) || defined(USE_WEBGPU) TEST(TensorOpTest, TileMLFloat16Type) { RunTestWrapper(); } #endif diff --git a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc index 73a5bce768a2a..00449cd442a32 100644 --- a/onnxruntime/test/providers/cpu/tensor/transpose_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/transpose_test.cc @@ -771,11 +771,9 @@ TEST(TransposeOpTest, DoTransposeEltWise) { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) static void TestTranspose( const std::vector& perm, const std::vector& x_dims, @@ -867,7 +865,7 @@ TEST(TransposeOpTest, TransposeBigMLFloat16) { // Exercises CanUse_cublasTransp const std::vector Y_dims{1, 1449, 1449, 3}; TestTransposeMLFloat16(perm, X_dims, Y_dims); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc index 3ac8053aef95e..10dd14be4ce92 100644 --- a/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/upsample_op_test.cc @@ -91,9 +91,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearestTest) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearestTest_int32) { @@ -174,10 +173,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearestTest_int32) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support - // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearestTest_uint8) { @@ -259,9 +256,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearestTest_uint8) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearest2XTest) { @@ -335,9 +331,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearest2XTest) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearest222XTest) { @@ -441,9 +436,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearest222XTest) { test.AddOutput("Y", {(int64_t)(N * scales[0]), (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearest15XTest) { @@ -513,9 +507,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearest15XTest) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearestTest_NoScale) { @@ -615,9 +608,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearest2XTest_int32) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOp4DBilinearTest) { @@ -691,9 +683,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4D1CBilinearTest) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest) { @@ -765,9 +756,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOp2DBilinearTest) { @@ -885,9 +875,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOp4DBilinearTest_int32) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } TEST(UpsampleOpTest, UpsampleOpNearestTest_1D) { @@ -985,9 +974,8 @@ TEST(UpsampleOpTest, NhwcUpsampleOpNearest2XTest_opset9) { test.AddOutput("Y", {N, (int64_t)(H * scales[1]), (int64_t)(W * scales[2]), C}, Y); // CUDA: result mismatch due to not implementing NHWC support // TensorRT: results mismatch - // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", - {kCudaExecutionProvider, kTensorrtExecutionProvider, kRocmExecutionProvider}); + {kCudaExecutionProvider, kTensorrtExecutionProvider}); } } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/kernel_compute_test_utils.cc b/onnxruntime/test/providers/kernel_compute_test_utils.cc index 93e688570631e..9f75797936f03 100644 --- a/onnxruntime/test/providers/kernel_compute_test_utils.cc +++ b/onnxruntime/test/providers/kernel_compute_test_utils.cc @@ -32,15 +32,6 @@ void KernelComputeTester::Run(std::unordered_set strided_outputs) { ASSERT_STATUS_OK(execution_providers.Add(ep_type, std::move(cuda_ep))); } #endif -#ifdef USE_ROCM - if (provider_ == kRocmExecutionProvider) { - auto rocm_ep = DefaultRocmExecutionProvider(); - ep_type = rocm_ep->Type(); - auto rocm_transfer = rocm_ep->GetDataTransfer(); - ASSERT_STATUS_OK(dtm.RegisterDataTransfer(std::move(rocm_transfer))); - ASSERT_STATUS_OK(execution_providers.Add(ep_type, std::move(rocm_ep))); - } -#endif const auto& logger = DefaultLoggingManager().DefaultLogger(); Model model("test", false, ModelMetaData(), ORT_TSTR(""), IOnnxRuntimeOpSchemaRegistryList(), @@ -56,8 +47,8 @@ void KernelComputeTester::Run(std::unordered_set strided_outputs) { if (provider_ == kCpuExecutionProvider || data.is_cpu_data_) { initializer_map[name] = data.value_; } -#if defined(USE_CUDA) || defined(USE_ROCM) - if ((provider_ == kCudaExecutionProvider || provider_ == kRocmExecutionProvider) && !data.is_cpu_data_) { +#if defined(USE_CUDA) + if (provider_ == kCudaExecutionProvider && !data.is_cpu_data_) { const Tensor& tensor = data.value_.Get(); Tensor gpu_tensor(tensor.DataType(), tensor.Shape(), diff --git a/onnxruntime/test/python/onnxruntime_test_float8_gemm8.py b/onnxruntime/test/python/onnxruntime_test_float8_gemm8.py index bb65533c3d1e0..f16415d625b5d 100644 --- a/onnxruntime/test/python/onnxruntime_test_float8_gemm8.py +++ b/onnxruntime/test/python/onnxruntime_test_float8_gemm8.py @@ -139,11 +139,6 @@ def common_test_model_gemm( providers = ["CPUExecutionProvider"] if "CUDAExecutionProvider" in available_providers: providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] - elif "ROCMExecutionProvider" in available_providers: - providers = [ - ("ROCMExecutionProvider", {"tunable_op_enable": "1", "tunable_op_tuning_enable": "1"}), - ("CPUExecutionProvider", {}), - ] expected = (a.T if kwargs.get("transA", 0) else a) @ (b.T if kwargs.get("transB", 0) else b) expected *= kwargs.get("alpha", 1.0) @@ -341,29 +336,6 @@ def test_combinations(self, shapeA, shapeB, transA, transB): self.assertEqual(expected.dtype, got[0].dtype) assert_allclose(expected, got[0]) - @parameterized.parameterized.expand( - [ - ("FLOAT8E4M3FN", "FLOAT16", 0, 0), - ("FLOAT16", "FLOAT8E4M3FN", 0, 0), - ("FLOAT16", "FLOAT8E4M3FN", 0, 1), - ] - ) - @unittest.skipIf("ROCMExecutionProvider" not in available_providers, reason="Not running without ROCm.") - @unittest.skipIf(not hasattr(TensorProto, "FLOAT8E4M3FN"), reason="needs onnx>=1.14.0") - def test_model_rocm_gemm_float8_e4m3(self, a_float_name, b_float_name, transA, transB): - self.common_test_model_gemm( - a_float_name=a_float_name, - b_float_name=b_float_name, - c_float_name="FLOAT8E4M3FN", - rtol=0.5, - dtype=TensorProto.FLOAT16, - transA=0, - transB=transB, - scaleY=False, - alpha=10.0, - beta=0.0, - ) - if __name__ == "__main__": # TestFloat8Gemm8().test_model_gemm_float() diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index 7f003453add89..768a97d7ed2bc 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -542,47 +542,6 @@ def run_advanced_test(cuda_lib): print("run advanced_test") run_advanced_test(cuda) - if "ROCMExecutionProvider" in onnxrt.get_available_providers(): - - def run_rocm_options_test(): - sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), providers=["ROCMExecutionProvider"]) - self.assertIn("ROCMExecutionProvider", sess.get_providers()) - options = sess.get_provider_options() - - def test_get_and_set_option_with_values(option_name, option_values): - provider_options = sess.get_provider_options() - self.assertIn("ROCMExecutionProvider", provider_options) - rocm_options = options["ROCMExecutionProvider"] - self.assertIn(option_name, rocm_options) - for option_value in option_values: - rocm_options[option_name] = option_value - sess.set_providers(["ROCMExecutionProvider"], [rocm_options]) - new_provider_options = sess.get_provider_options() - self.assertEqual( - new_provider_options.get("ROCMExecutionProvider", {}).get(option_name), - str(option_value), - ) - - test_get_and_set_option_with_values("tunable_op_enable", ["1", "0"]) - - test_get_and_set_option_with_values("tunable_op_tuning_enable", ["1", "0"]) - - test_get_and_set_option_with_values("tunable_op_max_tuning_duration_ms", ["-1", "1"]) - - test_get_and_set_option_with_values("enable_hip_graph", ["1", "0"]) - - # test for user_compute_stream - option = options["ROCMExecutionProvider"] - option["user_compute_stream"] = "1" - sess.set_providers(["ROCMExecutionProvider"], [option]) - new_options = sess.get_provider_options() - new_option = new_options["ROCMExecutionProvider"] - self.assertEqual(new_option["user_compute_stream"], "1") - # set user_compute_stream will set has_user_compute_stream to 1 too - self.assertEqual(new_option["has_user_compute_stream"], "1") - - run_rocm_options_test() - def test_invalid_set_providers(self): with self.assertRaises(RuntimeError) as context: sess = onnxrt.InferenceSession(get_name("mul_1.onnx"), providers=["CPUExecutionProvider"]) @@ -675,9 +634,6 @@ def do_test_get_and_set_tuning_results(ep): if "CUDAExecutionProvider" in onnxrt.get_available_providers(): do_test_get_and_set_tuning_results("CUDAExecutionProvider") - if "ROCMExecutionProvider" in onnxrt.get_available_providers(): - do_test_get_and_set_tuning_results("ROCMExecutionProvider") - def test_run_model_with_optional_sequence_input(self): sess = onnxrt.InferenceSession(get_name("identity_opt.onnx")) x = [np.array([1, 2, 3, 4, 5]).astype(np.float32)] @@ -1799,11 +1755,6 @@ def check_failure(providers, provider_options): check_failure([("a", {1: 2})], [{3: 4}]) def test_register_custom_e_ps_library(self): - available_eps = C.get_available_providers() - # skip amd gpu build - if "ROCMExecutionProvider" in available_eps: - return - if sys.platform.startswith("win"): shared_library = os.path.abspath("test_execution_provider.dll") diff --git a/onnxruntime/test/python/transformers/parity_utilities.py b/onnxruntime/test/python/transformers/parity_utilities.py index 7066d5a5425cb..fa16f0e67a523 100644 --- a/onnxruntime/test/python/transformers/parity_utilities.py +++ b/onnxruntime/test/python/transformers/parity_utilities.py @@ -181,8 +181,6 @@ def create_ort_session(onnx_model_path, use_gpu=True, optimized=True, verbose=Fa if not optimized: execution_providers.append("MIGraphXExecutionProvider") - execution_providers.append("ROCMExecutionProvider") - execution_providers.append("CPUExecutionProvider") return InferenceSession(onnx_model_path, sess_options, providers=execution_providers) diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index c4a92e959b273..a39b9c89e5898 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -47,10 +47,6 @@ #include #endif -#ifdef USE_ROCM -#include -#endif - #ifdef USE_DML #include #include @@ -294,14 +290,6 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod session_options.AppendExecutionProvider_Dnnl(dnnl_options); #else return; -#endif - } else if (provider_type == 3) { -#ifdef USE_ROCM - std::cout << "Running simple inference with rocm provider" << std::endl; - OrtROCMProviderOptions rocm_options; - session_options.AppendExecutionProvider_ROCM(rocm_options); -#else - return; #endif } else { std::cout << "Running simple inference with default provider" << std::endl; @@ -347,7 +335,7 @@ static void TestInference(Ort::Env& env, const std::basic_string& mod } static constexpr PATH_TYPE MODEL_URI = TSTR("testdata/mul_1.onnx"); -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_DML) static constexpr PATH_TYPE CUDA_GRAPH_ANNOTATION_MODEL_URI = TSTR("testdata/mul_1_dynamic.onnx"); #endif static constexpr PATH_TYPE MATMUL_MODEL_URI = TSTR("testdata/matmul_1.onnx"); @@ -1715,15 +1703,12 @@ TEST(CApiTest, test_custom_op_library) { #ifdef USE_CUDA TestInference(*ort_env, CUSTOM_OP_LIBRARY_TEST_MODEL_URI, inputs, "output", expected_dims_y, expected_values_y, 1, nullptr, lib_name.c_str()); -#elif USE_ROCM - TestInference(*ort_env, CUSTOM_OP_LIBRARY_TEST_MODEL_URI, inputs, "output", expected_dims_y, - expected_values_y, 3, nullptr, lib_name.c_str()); #elif USE_DML TestInference(*ort_env, CUSTOM_OP_LIBRARY_TEST_MODEL_URI, inputs, "output", expected_dims_y, expected_values_y, 4, nullptr, lib_name.c_str()); #else -TestInference(*ort_env, CUSTOM_OP_LIBRARY_TEST_MODEL_URI, inputs, "output", expected_dims_y, - expected_values_y, 0, nullptr, lib_name.c_str()); + TestInference(*ort_env, CUSTOM_OP_LIBRARY_TEST_MODEL_URI, inputs, "output", expected_dims_y, + expected_values_y, 0, nullptr, lib_name.c_str()); #endif } @@ -2103,27 +2088,6 @@ TEST(CApiTest, get_allocator_cuda) { } #endif -#ifdef USE_ROCM -TEST(CApiTest, get_allocator_rocm) { - Ort::SessionOptions session_options; - Ort::ThrowOnError(OrtSessionOptionsAppendExecutionProvider_ROCM(session_options, 0)); - Ort::Session session(*ort_env, NAMED_AND_ANON_DIM_PARAM_URI, session_options); - - Ort::MemoryInfo info_rocm("Hip", OrtAllocatorType::OrtArenaAllocator, 0, OrtMemTypeDefault); - Ort::Allocator rocm_allocator(session, info_rocm); - - auto allocator_info = rocm_allocator.GetInfo(); - ASSERT_TRUE(info_rocm == allocator_info); - void* p = rocm_allocator.Alloc(1024); - ASSERT_NE(p, nullptr); - rocm_allocator.Free(p); - - auto mem_allocation = rocm_allocator.GetAllocation(1024); - ASSERT_NE(nullptr, mem_allocation.get()); - ASSERT_EQ(1024U, mem_allocation.size()); -} -#endif - #if defined(USE_QNN) TEST(CApiTest, get_allocator_qnn_htp_shared) { @@ -2424,7 +2388,7 @@ TEST(CApiTest, io_binding_qnn_htp_shared) { #endif // defined(USE_QNN) -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_DML) TEST(CApiTest, basic_cuda_graph) { [[maybe_unused]] const auto& api = Ort::GetApi(); Ort::SessionOptions session_options; @@ -2445,19 +2409,6 @@ TEST(CApiTest, basic_cuda_graph) { cuda_options.Update(options_map); session_options.AppendExecutionProvider_CUDA_V2(*cuda_options); -#elif defined(USE_ROCM) - // Enable hip graph in rocm provider option. - OrtROCMProviderOptions* rocm_options = nullptr; - ASSERT_TRUE(api.CreateROCMProviderOptions(&rocm_options) == nullptr); - std::unique_ptr - rel_rocm_options(rocm_options, api.ReleaseROCMProviderOptions); - std::vector keys{"enable_hip_graph"}; - std::vector values{"1"}; - ASSERT_TRUE(api.UpdateROCMProviderOptions(rel_rocm_options.get(), keys.data(), values.data(), 1) == nullptr); - - ASSERT_TRUE(api.SessionOptionsAppendExecutionProvider_ROCM( - static_cast(session_options), - rel_rocm_options.get()) == nullptr); #elif defined(USE_DML) // Enable dynamic DML graph in DML provider option. session_options.AddConfigEntry("ep.dml.enable_graph_capture", "1"); @@ -2469,13 +2420,7 @@ TEST(CApiTest, basic_cuda_graph) { #endif Ort::Session session(*ort_env, MODEL_URI, session_options); -#if defined(USE_ROCM) -// local hipify -#define cudaMemcpy hipMemcpy -#define cudaMemcpyHostToDevice hipMemcpyHostToDevice -#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost - Ort::MemoryInfo info_mem("Hip", OrtAllocatorType::OrtArenaAllocator, 0, OrtMemTypeDefault); -#elif defined(USE_CUDA) || defined(USE_TENSORRT) +#if defined(USE_CUDA) || defined(USE_TENSORRT) Ort::MemoryInfo info_mem("Cuda", OrtAllocatorType::OrtArenaAllocator, 0, OrtMemTypeDefault); #elif defined(USE_DML) Ort::MemoryInfo info_mem("DML", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemTypeDefault); @@ -2491,7 +2436,7 @@ TEST(CApiTest, basic_cuda_graph) { ASSERT_NE(input_data.get(), nullptr); -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_TENSORRT) (void)cudaMemcpy(input_data.get(), x_values.data(), sizeof(float) * x_values.size(), cudaMemcpyHostToDevice); #elif defined(USE_DML) ComPtr input_resource; @@ -2524,7 +2469,7 @@ TEST(CApiTest, basic_cuda_graph) { // Check the values against the bound raw memory (needs copying from device to host first) std::array y_values; -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_TENSORRT) (void)cudaMemcpy(y_values.data(), output_data.get(), sizeof(float) * y_values.size(), cudaMemcpyDeviceToHost); #elif defined(USE_DML) ComPtr output_resource; @@ -2538,7 +2483,7 @@ TEST(CApiTest, basic_cuda_graph) { // Replay the captured CUDA graph session.Run(Ort::RunOptions(), binding); -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_TENSORRT) (void)cudaMemcpy(y_values.data(), output_data.get(), sizeof(float) * y_values.size(), cudaMemcpyDeviceToHost); #elif defined(USE_DML) DownloadDataFromDml(dml_objects, output_resource.Get(), gsl::make_span(output_cpu_bytes, sizeof(float) * y_values.size())); @@ -2549,7 +2494,7 @@ TEST(CApiTest, basic_cuda_graph) { // Change the input and replay the CUDA graph again. x_values = {10.0f, 20.0f, 30.0f, 40.0f, 50.0f, 60.0f}; -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_TENSORRT) (void)cudaMemcpy(input_data.get(), x_values.data(), sizeof(float) * x_values.size(), cudaMemcpyHostToDevice); #elif defined(USE_DML) UploadDataToDml(dml_objects, input_resource.Get(), gsl::make_span(reinterpret_cast(x_values.data()), sizeof(float) * x_values.size())); @@ -2559,7 +2504,7 @@ TEST(CApiTest, basic_cuda_graph) { session.Run(Ort::RunOptions(), binding); -#if defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_TENSORRT) (void)cudaMemcpy(y_values.data(), output_data.get(), sizeof(float) * y_values.size(), cudaMemcpyDeviceToHost); #elif defined(USE_DML) DownloadDataFromDml(dml_objects, output_resource.Get(), gsl::make_span(output_cpu_bytes, sizeof(float) * y_values.size())); @@ -2571,14 +2516,9 @@ TEST(CApiTest, basic_cuda_graph) { // Clean up binding.ClearBoundInputs(); binding.ClearBoundOutputs(); -#if defined(USE_ROCM) -#undef cudaMemcpy -#undef cudaMemcpyHostToDevice -#undef cudaMemcpyDeviceToHost -#endif } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) +#if defined(USE_CUDA) || defined(USE_DML) struct CudaGraphInputOutputData_0 { const std::array x_shape = {3, 2}; std::array x_values = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; @@ -2622,12 +2562,6 @@ static void RunWithCudaGraphAnnotation(T& cg_data, Ort::MemoryAllocation& input_data, Ort::MemoryAllocation& output_data, const char* cuda_graph_annotation) { -// a local hipify of select cuda symbols to avoid code duplication -#ifdef USE_ROCM -#define cudaMemcpy hipMemcpy -#define cudaMemcpyHostToDevice hipMemcpyHostToDevice -#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost -#endif #ifdef USE_DML Ort::SessionOptions session_options; Ort::Allocator allocator(session, info_mem); @@ -2731,11 +2665,6 @@ static void RunWithCudaGraphAnnotation(T& cg_data, // Clean up binding.ClearBoundInputs(); binding.ClearBoundOutputs(); -#ifdef USE_ROCM -#undef cudaMemcpy -#undef cudaMemcpyHostToDevice -#undef cudaMemcpyDeviceToHost -#endif } TEST(CApiTest, basic_cuda_graph_with_annotation) { @@ -2758,20 +2687,6 @@ TEST(CApiTest, basic_cuda_graph_with_annotation) { session_options.AppendExecutionProvider_CUDA_V2(*cuda_options); Ort::MemoryInfo info_mem("Cuda", OrtAllocatorType::OrtArenaAllocator, 0, OrtMemTypeDefault); -#elif defined(USE_ROCM) - // Enable hip graph in rocm provider option. - OrtROCMProviderOptions* rocm_options = nullptr; - ASSERT_TRUE(api.CreateROCMProviderOptions(&rocm_options) == nullptr); - std::unique_ptr - rel_rocm_options(rocm_options, api.ReleaseROCMProviderOptions); - std::vector keys{"enable_hip_graph"}; - std::vector values{"1"}; - ASSERT_TRUE(api.UpdateROCMProviderOptions(rel_rocm_options.get(), keys.data(), values.data(), 1) == nullptr); - - ASSERT_TRUE(api.SessionOptionsAppendExecutionProvider_ROCM( - static_cast(session_options), - rel_rocm_options.get()) == nullptr); - Ort::MemoryInfo info_mem("Hip", OrtAllocatorType::OrtArenaAllocator, 0, OrtMemTypeDefault); #endif Ort::Session session(*ort_env, CUDA_GRAPH_ANNOTATION_MODEL_URI, session_options); @@ -2820,34 +2735,10 @@ TEST(CApiTest, cuda_graph_with_shape_nodes) { } #endif // defined(USE_CUDA) || defined(USE_TENSORRT) -#if defined(USE_ROCM) -TEST(CApiTest, hip_graph_with_shape_nodes) { - const auto& api = Ort::GetApi(); - - // Enable hip graph in rocm provider option. - OrtROCMProviderOptions* rocm_options = nullptr; - ASSERT_TRUE(api.CreateROCMProviderOptions(&rocm_options) == nullptr); - std::unique_ptr - rel_rocm_options(rocm_options, api.ReleaseROCMProviderOptions); - std::vector keys{"enable_hip_graph"}; - std::vector values{"1"}; - ASSERT_TRUE(api.UpdateROCMProviderOptions(rel_rocm_options.get(), keys.data(), values.data(), 1) == nullptr); - - Ort::SessionOptions session_options; - ASSERT_TRUE(api.SessionOptionsAppendExecutionProvider_ROCM( - static_cast(session_options), - rel_rocm_options.get()) == nullptr); - - // Successful loading of the ONNX model with shape nodes with hip graph feature enabled - Ort::Session session(*ort_env, TSTR("testdata/cuda_graph_with_shape_nodes.onnx"), session_options); -} -#endif // defined(USE_ROCM) - #if defined(USE_DML) TEST(CApiTest, dml_graph_with_shape_nodes) { const auto& api = Ort::GetApi(); - // Enable hip graph in rocm provider option. const OrtDmlApi* ort_dml_api; Ort::SessionOptions session_options; session_options.AddConfigEntry("ep.dml.enable_graph_capture", "1"); @@ -2861,7 +2752,7 @@ TEST(CApiTest, dml_graph_with_shape_nodes) { #endif // REDUCED_OPS_BUILD -#endif // defined(USE_CUDA) || defined(USE_TENSORRT) || defined(USE_ROCM) +#endif // defined(USE_CUDA) || defined(USE_TENSORRT) TEST(CApiTest, create_tensor) { const char* s[] = {"abc", "kmp"}; diff --git a/onnxruntime/test/unittest_util/base_tester.cc b/onnxruntime/test/unittest_util/base_tester.cc index 4d640e0f5e33d..0887c200e49f0 100644 --- a/onnxruntime/test/unittest_util/base_tester.cc +++ b/onnxruntime/test/unittest_util/base_tester.cc @@ -666,7 +666,6 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter, kArmNNExecutionProvider, kNnapiExecutionProvider, kVSINPUExecutionProvider, - kRocmExecutionProvider, kCoreMLExecutionProvider, kCoreMLExecutionProviderMLProgram, kQnnExecutionProvider, @@ -732,8 +731,6 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter, execution_provider = DefaultAclExecutionProvider(); else if (provider_type == onnxruntime::kArmNNExecutionProvider) execution_provider = DefaultArmNNExecutionProvider(); - else if (provider_type == onnxruntime::kRocmExecutionProvider) - execution_provider = DefaultRocmExecutionProvider(); else if (provider_type == onnxruntime::kCoreMLExecutionProvider) execution_provider = DefaultCoreMLExecutionProvider(); else if (provider_type == kCoreMLExecutionProviderMLProgram) @@ -771,27 +768,6 @@ void BaseTester::RunWithConfig(size_t* number_of_pre_packed_weights_counter, number_of_pre_packed_weights_counter, number_of_shared_pre_packed_weights_counter); - // Run Models with subscribed run_options->config_options - if (ctx_.run_options != nullptr && - ctx_.run_options->config_options.GetConfigEntry(kOpTesterRunOptionsConfigTestTunableOp) == "true") { - std::vector> execution_providers; - if (provider_type == onnxruntime::kRocmExecutionProvider) { - execution_providers.emplace_back(DefaultRocmExecutionProvider(/*test_tunable_op=*/true)); - } - - if (!execution_providers.empty()) { - ExecuteModelForEps( - std::move(execution_providers), model, ctx_.session_options, - ctx_.expect_result, ctx_.expected_failure_string, - ctx_.run_options, feeds, output_names, - &custom_session_registries_, - /*assign_ep_for_nodes=*/true, - allow_released_onnx_opset_only, - number_of_pre_packed_weights_counter, - number_of_shared_pre_packed_weights_counter); - } - } - has_run = true; cur_provider = "not set"; } diff --git a/onnxruntime/test/util/default_providers.cc b/onnxruntime/test/util/default_providers.cc index cea3feeb927af..edccc314b75f0 100644 --- a/onnxruntime/test/util/default_providers.cc +++ b/onnxruntime/test/util/default_providers.cc @@ -329,9 +329,5 @@ std::unique_ptr DefaultDmlExecutionProvider() { return nullptr; } -std::unique_ptr DefaultRocmExecutionProvider(bool) { - return nullptr; -} - } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/util/include/default_providers.h b/onnxruntime/test/util/include/default_providers.h index ab3136c0b7b33..fb7b168f5e158 100644 --- a/onnxruntime/test/util/include/default_providers.h +++ b/onnxruntime/test/util/include/default_providers.h @@ -23,7 +23,6 @@ std::shared_ptr CreateExecutionProviderFactory_Nnapi( uint32_t flags, const optional& partitioning_stop_ops_list); std::shared_ptr CreateExecutionProviderFactory_VSINPU(); std::shared_ptr CreateExecutionProviderFactory_Rknpu(); -std::shared_ptr CreateExecutionProviderFactory_Rocm(const OrtROCMProviderOptions* provider_options); std::shared_ptr CreateExecutionProviderFactory_Tensorrt(const OrtTensorRTProviderOptions* params); std::shared_ptr CreateExecutionProviderFactory_Tensorrt(const OrtTensorRTProviderOptionsV2* params); std::shared_ptr CreateExecutionProviderFactory_Cann(const OrtCANNProviderOptions* provider_options); @@ -57,7 +56,6 @@ std::unique_ptr DefaultVSINPUExecutionProvider(); std::unique_ptr DefaultRknpuExecutionProvider(); std::unique_ptr DefaultAclExecutionProvider(bool enable_fast_math = false); std::unique_ptr DefaultArmNNExecutionProvider(bool enable_arena = true); -std::unique_ptr DefaultRocmExecutionProvider(bool test_tunable_op = false); std::unique_ptr DefaultCoreMLExecutionProvider(bool use_mlprogram = false); std::unique_ptr DefaultSnpeExecutionProvider(); std::unique_ptr DefaultQnnExecutionProvider(); diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc index a2b30eac03514..24703fd92dac5 100644 --- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc +++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc @@ -130,7 +130,7 @@ std::vector> GeneratePreTrainingTransformers( } transformers.emplace_back(std::make_unique(compatible_eps, level, true)); -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) transformers.emplace_back(std::make_unique(compatible_eps, true /* skip_device_check*/)); #else @@ -145,7 +145,7 @@ std::vector> GeneratePreTrainingTransformers( // Quantization Aware Training. So, replace QDQ nodes with FakeQuant. transformers.emplace_back(std::make_unique(compatible_eps)); -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) // We are supposed to use the execution provider as an indicator, // but here we don't have access to the registered EP at this point // as the session is not initialized yet. So using macro for now. @@ -180,8 +180,7 @@ std::vector> GeneratePreTrainingTransformers( config.number_recompute_layers, compatible_eps)); } if (config.propagate_cast_ops_config.level >= 0) { - const InlinedHashSet cuda_execution_provider = {onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider}; + const InlinedHashSet cuda_execution_provider = {onnxruntime::kCudaExecutionProvider}; transformers.emplace_back(std::make_unique(config.propagate_cast_ops_config.strategy, static_cast(config.propagate_cast_ops_config.level), config.propagate_cast_ops_config.allow, @@ -194,8 +193,8 @@ std::vector> GeneratePreTrainingTransformers( transformers.emplace_back(std::make_unique(compatible_eps)); transformers.emplace_back(std::make_unique(compatible_eps, config.print_input_density)); -#if defined(USE_CUDA) || defined(USE_ROCM) - // Put this under CUDA/ROCM guard as it depends on PadAndUnflatten CUDA/ROCM kernel. +#if defined(USE_CUDA) + // Put this under CUDA guard as it depends on PadAndUnflatten CUDA kernel. // Once we have a CPU kernel for PadAndUnflatten, we can remove the guard. transformers.emplace_back(std::make_unique(compatible_eps, config.print_input_density)); @@ -261,17 +260,16 @@ InlinedVector> GenerateTransformers( switch (level) { case TransformerLevel::Level1: { InlinedHashSet l1_execution_providers = {}; - InlinedHashSet cuda_rocm_execution_providers = {onnxruntime::kCudaExecutionProvider, - onnxruntime::kRocmExecutionProvider}; + InlinedHashSet cuda_execution_providers = {onnxruntime::kCudaExecutionProvider}; // TODO hack - constant folding currently doesn't work after mixed precision transformation so it's disabled for now // ORT uses CPU kernels to evaluate constant values but some of them don't support fp16 // transformers.emplace_back(std::make_unique(l1_execution_providers)); transformers.emplace_back(std::make_unique(l1_execution_providers)); transformers.emplace_back(std::make_unique(free_dimension_overrides)); - transformers.emplace_back(std::make_unique(cuda_rocm_execution_providers)); - transformers.emplace_back(std::make_unique(cuda_rocm_execution_providers)); - transformers.emplace_back(std::make_unique(cuda_rocm_execution_providers)); + transformers.emplace_back(std::make_unique(cuda_execution_providers)); + transformers.emplace_back(std::make_unique(cuda_execution_providers)); + transformers.emplace_back(std::make_unique(cuda_execution_providers)); transformers.emplace_back(std::make_unique(l1_execution_providers)); InlinedHashSet excluded_initializers(weights_to_train.begin(), weights_to_train.end()); transformers.emplace_back(std::make_unique(l1_execution_providers, excluded_initializers)); diff --git a/orttraining/orttraining/models/bert/main.cc b/orttraining/orttraining/models/bert/main.cc index c4c7a98ba116a..772c1ef5d856a 100644 --- a/orttraining/orttraining/models/bert/main.cc +++ b/orttraining/orttraining/models/bert/main.cc @@ -26,11 +26,6 @@ namespace onnxruntime { std::unique_ptr CreateCUDAPinnedAllocator(const char* name); } // namespace onnxruntime #endif -#ifdef USE_ROCM -namespace onnxruntime { -std::unique_ptr CreateROCMPinnedAllocator(const char* name); -} // namespace onnxruntime -#endif using namespace onnxruntime; using namespace onnxruntime::common; @@ -638,22 +633,6 @@ void setup_training_params(BertParameters& params) { } #endif -#ifdef USE_ROCM - { - OrtROCMProviderOptions info; - info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); - info.do_copy_in_default_stream = true; - - if (params.gpu_mem_limit_in_gb > 0) { - info.gpu_mem_limit = gsl::narrow(params.gpu_mem_limit_in_gb * 1024 * 1024 * 1024); - } - info.miopen_conv_exhaustive_search = true; // true, exhaustive search (slow) - - params.providers.emplace(kRocmExecutionProvider, RocmProviderFactoryCreator::Create(&info)); - params.input_allocator = CreateROCMPinnedAllocator(HIP_PINNED); - } -#endif - params.loss_func_info = LossFunctionInfo(OpDef("BertLoss", kOnnxDomain), "total_loss", {/*prediction_masked_lm*/ "output1", diff --git a/orttraining/orttraining/models/gpt2/main.cc b/orttraining/orttraining/models/gpt2/main.cc index 165d69fb1378a..b1e6222d1fb19 100644 --- a/orttraining/orttraining/models/gpt2/main.cc +++ b/orttraining/orttraining/models/gpt2/main.cc @@ -26,11 +26,6 @@ namespace onnxruntime { std::unique_ptr CreateCUDAPinnedAllocator(const char* name); } // namespace onnxruntime #endif -#ifdef USE_ROCM -namespace onnxruntime { -std::unique_ptr CreateROCMPinnedAllocator(const char* name); -} // namespace onnxruntime -#endif using namespace onnxruntime; using namespace onnxruntime::common; @@ -368,16 +363,6 @@ void setup_training_params(GPT2Parameters& params) { } #endif -#ifdef USE_ROCM - { - OrtROCMProviderOptions info; - info.device_id = gsl::narrow(MPIContext::GetInstance().GetLocalRank()); - info.do_copy_in_default_stream = true; - params.providers.emplace(kRocmExecutionProvider, RocmProviderFactoryCreator::Create(&info)); - params.input_allocator = CreateROCMPinnedAllocator(HIP_PINNED); - } -#endif - params.use_nccl = true; params.error_function = [params](const std::vector& /*feed_names*/, diff --git a/orttraining/orttraining/models/runner/training_runner.h b/orttraining/orttraining/models/runner/training_runner.h index f870f445ead91..da2ebb67cb52e 100644 --- a/orttraining/orttraining/models/runner/training_runner.h +++ b/orttraining/orttraining/models/runner/training_runner.h @@ -138,8 +138,7 @@ class TrainingRunner { } bool UseCuda() const { - return providers.find(kCudaExecutionProvider) != providers.end() || - providers.find(kRocmExecutionProvider) != providers.end(); + return providers.find(kCudaExecutionProvider) != providers.end(); } AdasumReductionType GetAdasumReductionType() const { diff --git a/orttraining/orttraining/python/orttraining_python_module.cc b/orttraining/orttraining/python/orttraining_python_module.cc index 1aa8d2090b511..3d611a0881fdf 100644 --- a/orttraining/orttraining/python/orttraining_python_module.cc +++ b/orttraining/orttraining/python/orttraining_python_module.cc @@ -33,11 +33,6 @@ const CUDAExecutionProviderInfo GetCudaExecutionProviderInfo(ProviderInfo_CUDA* const ProviderOptionsMap& provider_options_map); #endif -#ifdef USE_ROCM -const ROCMExecutionProviderInfo GetRocmExecutionProviderInfo(ProviderInfo_ROCM* rocm_provider_info, - const ProviderOptionsMap& provider_options_map); -#endif - void addGlobalMethods(py::module& m); void addObjectMethods(py::module& m, ExecutionProviderRegistrationFn ep_registration_fn); void addObjectMethodsForTraining(py::module& m); @@ -77,7 +72,7 @@ bool GetDynamicExecutionProviderHash( bool GetProviderInstanceHash(const std::string& type, const ProviderOptionsMap& provider_options_map, size_t& hash) { - // for built-in execution provider, currently only cpu / cuda / rocm support hash. + // for built-in execution provider, currently only cpu / cuda support hash. if (type == kCpuExecutionProvider) { // for CPU, only 1 instance hash = 0; @@ -90,15 +85,6 @@ bool GetProviderInstanceHash(const std::string& type, hash = std::hash{}(info); return true; } -#endif - } else if (type == kRocmExecutionProvider) { -#ifdef USE_ROCM - if (auto* rocm_provider_info = TryGetProviderInfo_ROCM()) { - const ROCMExecutionProviderInfo info = GetRocmExecutionProviderInfo(rocm_provider_info, - provider_options_map); - hash = std::hash{}(info); - return true; - } #endif } else { const auto it = provider_options_map.find(type); diff --git a/orttraining/orttraining/python/training/ortmodule/__init__.py b/orttraining/orttraining/python/training/ortmodule/__init__.py index 4bc470c633437..4c27fd923c96b 100644 --- a/orttraining/orttraining/python/training/ortmodule/__init__.py +++ b/orttraining/orttraining/python/training/ortmodule/__init__.py @@ -138,7 +138,6 @@ def _checkpoint( ORTMODULE_IS_DETERMINISTIC = torch.are_deterministic_algorithms_enabled() ONNXRUNTIME_CUDA_VERSION = ort_info.cuda_version if hasattr(ort_info, "cuda_version") else None -ONNXRUNTIME_ROCM_VERSION = ort_info.rocm_version if hasattr(ort_info, "rocm_version") else None # The first value indicates whether the code is in ONNX export context. # The export context here include the full export process, including prepare export input/output information, diff --git a/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py b/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py index d0447f1a96b17..0b7b338c27344 100755 --- a/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py +++ b/orttraining/orttraining/python/training/ortmodule/_graph_execution_manager.py @@ -10,7 +10,6 @@ import onnx import torch -from torch.utils.cpp_extension import ROCM_HOME import onnxruntime from onnxruntime.capi import _pybind_state as C @@ -91,8 +90,6 @@ def __init__( # To be instantiated in the concrete implementation of GraphExecutionManager self._export_mode = export_mode - self.is_rocm_pytorch = bool(torch.version.hip is not None and ROCM_HOME is not None) - # WIP feature to enable caching in Gradient accumulation scenario. self._gradient_accumulation_manager = GradientAccumulationManager() @@ -194,23 +191,22 @@ def _get_session_config(self): provider_options = None if self._device.type == "cuda": # Configure the InferenceSessions to use the specific GPU on which the model is placed. - providers = ["ROCMExecutionProvider"] if self.is_rocm_pytorch else ["CUDAExecutionProvider"] + providers = ["CUDAExecutionProvider"] providers.append("CPUExecutionProvider") provider_option_map = {"device_id": str(self._device.index)} - if not self.is_rocm_pytorch: - # Set Conv algo search mode to HEURISTIC by default, which is the same as PyTorch's default setting. - provider_option_map["cudnn_conv_algo_search"] = self._runtime_options.conv_algo_search - provider_option_map["cudnn_conv_use_max_workspace"] = "1" - provider_option_map["cudnn_conv1d_pad_to_nc1d"] = "1" - if self._runtime_options.enable_tuning: - provider_option_map["tunable_op_enable"] = "1" - provider_option_map["tunable_op_tuning_enable"] = "1" - if self._runtime_options.max_tuning_duration_ms: - provider_option_map["tunable_op_max_tuning_duration_ms"] = str( - self._runtime_options.max_tuning_duration_ms - ) - elif self._runtime_options.tuning_results_path: - provider_option_map["tunable_op_enable"] = "1" + # Set Conv algo search mode to HEURISTIC by default, which is the same as PyTorch's default setting. + provider_option_map["cudnn_conv_algo_search"] = self._runtime_options.conv_algo_search + provider_option_map["cudnn_conv_use_max_workspace"] = "1" + provider_option_map["cudnn_conv1d_pad_to_nc1d"] = "1" + if self._runtime_options.enable_tuning: + provider_option_map["tunable_op_enable"] = "1" + provider_option_map["tunable_op_tuning_enable"] = "1" + if self._runtime_options.max_tuning_duration_ms: + provider_option_map["tunable_op_max_tuning_duration_ms"] = str( + self._runtime_options.max_tuning_duration_ms + ) + elif self._runtime_options.tuning_results_path: + provider_option_map["tunable_op_enable"] = "1" if self._runtime_options.use_external_gpu_allocator: provider_option_map["gpu_external_alloc"] = str(self._torch_alloc) provider_option_map["gpu_external_free"] = str(self._torch_free) diff --git a/orttraining/orttraining/python/training/ortmodule/_training_manager.py b/orttraining/orttraining/python/training/ortmodule/_training_manager.py index b4303587e69e6..3fa3e1cdaf461 100644 --- a/orttraining/orttraining/python/training/ortmodule/_training_manager.py +++ b/orttraining/orttraining/python/training/ortmodule/_training_manager.py @@ -342,8 +342,6 @@ def _build_graph(self, graph_transformer_config): # Apply registered graph transformers to the optimized model device_type = self._device.type - if device_type == "cuda" and self.is_rocm_pytorch: - device_type = "rocm" GraphOptimizerRegistry.optimize_all( type(self._flattened_module._original_module).__name__, device_type, self._onnx_models.optimized_model.graph ) diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/__init__.py b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/__init__.py index e6b1f0fb8b391..561ebec1e7cc4 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/__init__.py +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/__init__.py @@ -19,7 +19,6 @@ The following environment variables are available for the extensions setup.py - ORTMODULE_TORCH_CPP_DIR: ORTModule's internal - - ONNXRUNTIME_ROCM_VERSION: ROCM version used to build ONNX Runtime package - ONNXRUNTIME_CUDA_VERSION: CUDA version used to build ONNX Runtime package - ONNXRUNTIME_FORCE_CUDA: Force CUDA extensions to be used when it is not available to build ONNX Runtime package diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/setup.py b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/setup.py index 6b028d8f05e11..ad3c298187ca3 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/setup.py +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/fused_ops/setup.py @@ -18,12 +18,10 @@ os.path.join(os.path.dirname(__file__), "multi_tensor_l2norm_kernel.cu"), ] -use_rocm = bool(os.environ["ONNXRUNTIME_ROCM_VERSION"]) extra_compile_args = {"cxx": ["-O3"]} -if not use_rocm: - nvcc_extra_args = os.environ.get("ONNXRUNTIME_CUDA_NVCC_EXTRA_ARGS", "") - if nvcc_extra_args: - extra_compile_args.update({"nvcc": nvcc_extra_args.split(",")}) +nvcc_extra_args = os.environ.get("ONNXRUNTIME_CUDA_NVCC_EXTRA_ARGS", "") +if nvcc_extra_args: + extra_compile_args.update({"nvcc": nvcc_extra_args.split(",")}) setup( name="fused_ops", diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/torch_gpu_allocator/setup.py b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/torch_gpu_allocator/setup.py index bdcb6daa233e6..68e15fb945361 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/torch_gpu_allocator/setup.py +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/cuda/torch_gpu_allocator/setup.py @@ -11,9 +11,9 @@ from torch.utils import cpp_extension # TODO: Implement a cleaner way to auto-generate torch_gpu_allocator.cc -use_rocm = bool(os.environ["ONNXRUNTIME_ROCM_VERSION"]) -gpu_identifier = "hip" if use_rocm else "cuda" -gpu_allocator_header = "HIPCachingAllocator" if use_rocm else "CUDACachingAllocator" + +gpu_identifier = "cuda" +gpu_allocator_header = "CUDACachingAllocator" filename = os.path.join(os.path.dirname(__file__), "torch_gpu_allocator.cc") with fileinput.FileInput(filename, inplace=True) as file: for line in file: @@ -24,10 +24,9 @@ sys.stdout.write(line) extra_compile_args = {"cxx": ["-O3"]} -if not use_rocm: - nvcc_extra_args = os.environ.get("ONNXRUNTIME_CUDA_NVCC_EXTRA_ARGS", "") - if nvcc_extra_args: - extra_compile_args.update({"nvcc": nvcc_extra_args.split(",")}) +nvcc_extra_args = os.environ.get("ONNXRUNTIME_CUDA_NVCC_EXTRA_ARGS", "") +if nvcc_extra_args: + extra_compile_args.update({"nvcc": nvcc_extra_args.split(",")}) setup( name="torch_gpu_allocator", diff --git a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py index b26259b8abf94..d36f0d872f4df 100644 --- a/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py +++ b/orttraining/orttraining/python/training/ortmodule/torch_cpp_extensions/install.py @@ -53,7 +53,7 @@ def build_torch_cpp_extensions(): """Builds PyTorch CPP extensions and returns metadata.""" # Run this from within onnxruntime package folder is_gpu_available = (torch.version.cuda is not None or torch.version.hip is not None) and ( - ortmodule.ONNXRUNTIME_CUDA_VERSION is not None or ortmodule.ONNXRUNTIME_ROCM_VERSION is not None + ortmodule.ONNXRUNTIME_CUDA_VERSION is not None ) # Docker build don't have CUDA support, but Torch C++ extensions with CUDA may be forced @@ -61,26 +61,23 @@ def build_torch_cpp_extensions(): os.chdir(ortmodule.ORTMODULE_TORCH_CPP_DIR) - # Extensions might leverage CUDA/ROCM versions internally + # Extensions might leverage CUDA versions internally os.environ["ONNXRUNTIME_CUDA_VERSION"] = ( ortmodule.ONNXRUNTIME_CUDA_VERSION if ortmodule.ONNXRUNTIME_CUDA_VERSION is not None else "" ) - os.environ["ONNXRUNTIME_ROCM_VERSION"] = ( - ortmodule.ONNXRUNTIME_ROCM_VERSION if ortmodule.ONNXRUNTIME_ROCM_VERSION is not None else "" - ) if torch.version.cuda is not None and ortmodule.ONNXRUNTIME_CUDA_VERSION is not None: _get_cuda_extra_build_params() ############################################################################ - # Pytorch CPP Extensions that DO require CUDA/ROCM + # Pytorch CPP Extensions that DO require CUDA ############################################################################ if is_gpu_available or force_cuda: for ext_setup in _list_cuda_extensions(): _install_extension(ext_setup.split(os.sep)[-2], ext_setup, ortmodule.ORTMODULE_TORCH_CPP_DIR) ############################################################################ - # Pytorch CPP Extensions that DO NOT require CUDA/ROCM + # Pytorch CPP Extensions that DO NOT require CUDA ############################################################################ for ext_setup in _list_cpu_extensions(): _install_extension(ext_setup.split(os.sep)[-2], ext_setup, ortmodule.ORTMODULE_TORCH_CPP_DIR) @@ -98,7 +95,6 @@ def build_torch_cpp_extensions(): # Tear down os.environ.pop("ONNXRUNTIME_CUDA_VERSION") - os.environ.pop("ONNXRUNTIME_ROCM_VERSION") if __name__ == "__main__": diff --git a/orttraining/orttraining/test/gradient/gradient_checker.cc b/orttraining/orttraining/test/gradient/gradient_checker.cc index b30540ec68317..0a837254fa619 100644 --- a/orttraining/orttraining/test/gradient/gradient_checker.cc +++ b/orttraining/orttraining/test/gradient/gradient_checker.cc @@ -42,8 +42,6 @@ std::vector> GetExecutionProviders( result.emplace_back(DefaultCpuExecutionProvider()); } else if (entry->Type() == onnxruntime::kCudaExecutionProvider) { result.emplace_back(DefaultCudaExecutionProvider()); - } else if (entry->Type() == onnxruntime::kRocmExecutionProvider) { - result.emplace_back(DefaultRocmExecutionProvider()); } else if (entry->Type() == onnxruntime::kDnnlExecutionProvider) { result.emplace_back(DefaultDnnlExecutionProvider()); } else if (entry->Type() == onnxruntime::kTensorrtExecutionProvider) { @@ -59,9 +57,6 @@ std::vector> GetExecutionProviders( } #ifdef USE_CUDA result.emplace_back(DefaultCudaExecutionProvider()); -#endif -#ifdef USE_ROCM - result.emplace_back(DefaultRocmExecutionProvider()); #endif result.emplace_back(DefaultCpuExecutionProvider()); return result; diff --git a/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc b/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc index 58c173ed90277..b362abbdde3d6 100644 --- a/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc +++ b/orttraining/orttraining/test/gradient/gradient_op_test_utils.cc @@ -59,7 +59,6 @@ void GradientOpTester::Run(int output_index_to_use_as_loss, static const std::string all_provider_types[] = { kCpuExecutionProvider, kCudaExecutionProvider, - kRocmExecutionProvider, kDnnlExecutionProvider, kTensorrtExecutionProvider, }; @@ -114,8 +113,7 @@ void GradientOpTester::Run(int output_index_to_use_as_loss, execution_provider = DefaultDnnlExecutionProvider(); else if (provider_type == onnxruntime::kTensorrtExecutionProvider) execution_provider = DefaultTensorrtExecutionProvider(); - else if (provider_type == onnxruntime::kRocmExecutionProvider) - execution_provider = DefaultRocmExecutionProvider(); + // skip if execution provider is disabled if (execution_provider == nullptr) continue; diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc index 2d0181b69413c..ae2b144bfdb69 100644 --- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc +++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc @@ -2231,7 +2231,6 @@ TEST(GradientUtilsTest, InPlaceAccumulatorV2Overwrite) { } #if defined(USE_CUDA) -// TODO: Add rocm kernel defs TEST(GradientUtilsTest, InPlaceAccumulatorV2_GPU) { std::vector> test_dims{ {768}, @@ -2276,7 +2275,7 @@ TEST(GradientUtilsTest, InPlaceAccumulatorV2_Float16) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(GradientUtilsTest, InPlaceAccumulatorFloat16) { OpTester test("InPlaceAccumulator", 1, onnxruntime::kMSDomain); @@ -2294,7 +2293,7 @@ TEST(GradientUtilsTest, InPlaceAccumulatorFloat16) { // Didn't implement mixed precision InPlaceAccumulator in CPU test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCpuExecutionProvider}); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(GradientUtilsTest, ZeroGradientFloat32) { OpTester test("ZeroGradient", 1, onnxruntime::kMSDomain); @@ -2307,7 +2306,7 @@ TEST(GradientUtilsTest, ZeroGradientFloat32) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(GradientUtilsTest, ZeroGradientFloat16) { OpTester test("ZeroGradient", 1, onnxruntime::kMSDomain); @@ -2327,7 +2326,7 @@ TEST(GradientUtilsTest, ZeroGradientFloat16) { test.Run(); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) TEST(GradientCheckerTest, WhereGrad) { float max_error; @@ -3019,7 +3018,6 @@ TEST(GradientCheckerTest, TriluGrad) { } } -// TODO (enable once found why it fails on ROCM) #if defined(USE_CUDA) TEST(GradientCheckerTest, PadAndUnflattenGrad) { float max_error; @@ -3035,8 +3033,6 @@ TEST(GradientCheckerTest, PadAndUnflattenGrad) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); #endif ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(op_def, {x_info, indices_info, shape_info}, @@ -3065,8 +3061,6 @@ TEST(GradientCheckerTest, ScaledSumGrad) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); #endif ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(op_def, {x_info, y_info}, @@ -3097,8 +3091,6 @@ TEST(GradientCheckerTest, ScaledSumGrad) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.emplace_back(DefaultRocmExecutionProvider()); #endif ASSERT_STATUS_OK(gradient_checker.ComputeGradientError(op_def, {x_info, y_info, z_info}, @@ -3318,7 +3310,6 @@ TEST(GradientCheckerTest, ConvTransposeGrad) { ConvTransposeGradientCheckerTest(&execution_providers); } -// TODO: Enable test for ROCM TEST(GradientCheckerTest, ResizeGrad) { std::vector> execution_providers; execution_providers.push_back(DefaultCudaExecutionProvider()); diff --git a/orttraining/orttraining/test/gradient/optimizer_ops_test.cc b/orttraining/orttraining/test/gradient/optimizer_ops_test.cc index 18c1364f5d1f6..96830510e8ebd 100644 --- a/orttraining/orttraining/test/gradient/optimizer_ops_test.cc +++ b/orttraining/orttraining/test/gradient/optimizer_ops_test.cc @@ -254,7 +254,7 @@ TEST(OptimizerTest, AdamWeightDecayMode1WithBiasCorrection) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) float GetGradientL2Norm(const std::vector& gradient_vector) { float gradient_norm = 0.0f; diff --git a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc index edabcb67aa586..1c2d71d4b4f90 100644 --- a/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc +++ b/orttraining/orttraining/test/graph/gradient_graph_builder_test.cc @@ -16,7 +16,7 @@ #include "orttraining/training_ops/cpu/controlflow/event_pool.h" // TODO: move with PipelineBatchPlanner -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include "bert_toy_fetches.h" #endif @@ -382,7 +382,7 @@ TEST(GradientGraphBuilderTest, TrainingSession_WithProfiler) { ASSERT_TRUE(count > 1); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) static void RunBertTrainingWithChecks( const SessionOptions& so, const PathString& backprop_model_file) { @@ -401,8 +401,6 @@ static void RunBertTrainingWithChecks( #ifdef USE_CUDA ASSERT_STATUS_OK(training_session->RegisterExecutionProvider(DefaultCudaExecutionProvider())); -#elif USE_ROCM - ASSERT_STATUS_OK(training_session->RegisterExecutionProvider(DefaultRocmExecutionProvider())); #endif ASSERT_STATUS_OK(training_session->Initialize()); @@ -579,7 +577,7 @@ TEST(GradientGraphBuilderTest, TrainingSession_BertToy) { PathString backprop_model_file; ASSERT_STATUS_OK(BuildBackPropGraph(model_path, config, backprop_model_file)); -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) SessionOptions so; RunBertTrainingWithChecks(so, backprop_model_file); #endif diff --git a/orttraining/orttraining/test/optimizer/compute_optimizer_test.cc b/orttraining/orttraining/test/optimizer/compute_optimizer_test.cc index d7f0f2ce9b743..2f7d6532c9cb1 100644 --- a/orttraining/orttraining/test/optimizer/compute_optimizer_test.cc +++ b/orttraining/orttraining/test/optimizer/compute_optimizer_test.cc @@ -559,8 +559,6 @@ TEST(ComputeOptimizerTests, InsertGatherBeforeSceLoss_MlmBertE2E) { onnxruntime::kCpuExecutionProvider, #ifdef USE_CUDA onnxruntime::kCudaExecutionProvider, -#elif USE_ROCM - onnxruntime::kRocmExecutionProvider, #endif }; diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index 7d5feb9742366..7b89b241b884e 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -831,7 +831,6 @@ def run_step(model, x): ort_model._is_training() )._execution_agent._inference_session._provider_options - # cudnn_conv_algo_search is for CUDA only, so setting the system env will not affect the compute on ROCm. if "CUDAExecutionProvider" in provider_options: expected_conv_algo_search = "HEURISTIC" if conv_algo_search is None else conv_algo_search actual_conv_algo_search = provider_options["CUDAExecutionProvider"]["cudnn_conv_algo_search"] @@ -6173,11 +6172,7 @@ def generate_inputs(batch_size, max_seq_length, vocab_size): for pt_param, ort_param in zip(pt_model.parameters(), ort_model.parameters(), strict=False): _test_helpers.assert_values_are_close(pt_param.grad, ort_param.grad, atol=1e-4, rtol=1e-5) - if os.getenv("ORTMODULE_ROCM_TEST", "0") == "1": - # For ROCm EP, the difference between ORT and PyTorch is larger than CUDA EP. - _test_helpers.assert_values_are_close(ort_prediction, pt_prediction, atol=2e-3, rtol=2e-4) - else: - _test_helpers.assert_values_are_close(ort_prediction, pt_prediction, atol=1e-3, rtol=1e-4) + _test_helpers.assert_values_are_close(ort_prediction, pt_prediction, atol=1e-3, rtol=1e-4) training_model = ort_model._torch_module._execution_manager(True)._onnx_models.optimized_model assert "FlattenAndUnpad" in [node.op_type for node in training_model.graph.node] @@ -6332,9 +6327,6 @@ def run_step(model, x): _test_helpers.assert_values_are_close(pt_x.grad, ort_x.grad) -@pytest.mark.skipif( - os.getenv("ORTMODULE_ROCM_TEST", "0") == "1", reason="Skip for ROCm because the kernel is not implemented for ROCm" -) @pytest.mark.parametrize("use_fp16", [False, True]) @pytest.mark.parametrize("conv_algo_search", [None, "EXHAUSTIVE", "HEURISTIC"]) def test_conv_transpose_gradient(use_fp16, conv_algo_search): @@ -6404,9 +6396,6 @@ def run_step(model, x): del os.environ["ORTMODULE_CONV_ALGO_SEARCH"] -@pytest.mark.skipif( - os.getenv("ORTMODULE_ROCM_TEST", "0") == "1", reason="Skip for ROCm because the kernel is not implemented for ROCm" -) @pytest.mark.parametrize("conv_algo_search", [None, "EXHAUSTIVE", "HEURISTIC"]) def test_conv_transpose_gradient_with_groups(conv_algo_search): class TransposedConv3DWithGroups(nn.Module): @@ -6450,9 +6439,6 @@ def run_step(model, x): del os.environ["ORTMODULE_CONV_ALGO_SEARCH"] -@pytest.mark.skipif( - os.getenv("ORTMODULE_ROCM_TEST", "0") == "1", reason="Skip for ROCm because the kernel is not implemented for ROCm" -) @pytest.mark.parametrize("conv_algo_search", [None, "EXHAUSTIVE", "HEURISTIC"]) def test_conv_transpose_gradient_with_strides_padding_and_dilation(conv_algo_search): class ConvTransposeComplexModel(nn.Module): @@ -6644,8 +6630,6 @@ def run_step(model, attn_weight): assert to_value == pytorch_type_to_onnx_dtype(softmax_compute_type), "Cast to attribute is not as expected" -# TODO: fix the issue in rocm training, then enable the test. -@pytest.mark.skip(reason="This test is disabled due to its breaking rocm training cis.") def test_aten_conv_bf16(): class NeuralNetConv(torch.nn.Module): def __init__(self): @@ -6920,11 +6904,7 @@ def generate_inputs(batch_size, max_seq_length, vocab_size): for ort_param1, ort_param2 in zip(ort_model1.parameters(), ort_model2.parameters(), strict=False): _test_helpers.assert_values_are_close(ort_param1.grad, ort_param2.grad, atol=1e-4, rtol=1e-5) - if os.getenv("ORTMODULE_ROCM_TEST", "0") == "1": - # For ROCm EP, the difference between ORT and PyTorch is larger than CUDA EP. - _test_helpers.assert_values_are_close(ort_prediction1, ort_prediction2, atol=2e-3, rtol=2e-4) - else: - _test_helpers.assert_values_are_close(ort_prediction1, ort_prediction2, atol=1e-3, rtol=1e-4) + _test_helpers.assert_values_are_close(ort_prediction1, ort_prediction2, atol=1e-3, rtol=1e-4) execution_mgr = ort_model2._torch_module._execution_manager._training_manager from onnxruntime.training.ortmodule._onnx_models import _get_onnx_file_name # noqa: PLC0415 diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_onnx_ops.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_onnx_ops.py index d977d96e82503..4ad615597b8b8 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_onnx_ops.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_onnx_ops.py @@ -151,7 +151,7 @@ def test_softmax_bf16_large(self): raise unittest.SkipTest("Temporarily disabled pending investigation") if torch.version.cuda is None: - # Only run this test when CUDA is available, as on ROCm BF16 is not supported by MIOpen. + # Only run this test when CUDA is available. return class Model(torch.nn.Module): diff --git a/orttraining/orttraining/test/session/training_session_test.cc b/orttraining/orttraining/test/session/training_session_test.cc index a3f6d917a76b6..e91c714165f0f 100644 --- a/orttraining/orttraining/test/session/training_session_test.cc +++ b/orttraining/orttraining/test/session/training_session_test.cc @@ -57,7 +57,7 @@ TEST(TrainingSessionTest, LoadOptimState_FullPrecision_FP32Moments_Adam) { RunTrainingSessionLoadOptimTests(k_adam_optimizer_op_name, false, false); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(TrainingSessionTest, LoadOptimState_MixedPrecision_FP32Moments_Adam) { RunTrainingSessionLoadOptimTests(k_adam_optimizer_op_name, true, false); } diff --git a/orttraining/orttraining/test/session/training_session_test_utils.cc b/orttraining/orttraining/test/session/training_session_test_utils.cc index 868388d4b9a93..07f375fa747e9 100644 --- a/orttraining/orttraining/test/session/training_session_test_utils.cc +++ b/orttraining/orttraining/test/session/training_session_test_utils.cc @@ -100,7 +100,7 @@ void VerifyState(const DataTransferManager& data_transfer_mgr, const NameMLValMa const auto& e_state_it = expected_state.find(key); ORT_ENFORCE(e_state_it != expected_state.end()); auto& expected_tensor = e_state_it->second.Get(); -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) auto& actual_gpu_tensor = a_state_it.second.Get(); // Copying tensor to CPU when cuda is enabled. @@ -181,8 +181,6 @@ std::unique_ptr BuildAndRunTrainingSessionWithChecks( #ifdef USE_CUDA ORT_THROW_IF_ERROR(training_session->RegisterExecutionProvider(DefaultCudaExecutionProvider())); -#elif USE_ROCM - ORT_THROW_IF_ERROR(training_session->RegisterExecutionProvider(DefaultRocmExecutionProvider())); #endif ORT_THROW_IF_ERROR(training_session->Initialize()); diff --git a/orttraining/orttraining/test/session/training_session_test_utils.h b/orttraining/orttraining/test/session/training_session_test_utils.h index 4ba092b951081..866855ef8d747 100644 --- a/orttraining/orttraining/test/session/training_session_test_utils.h +++ b/orttraining/orttraining/test/session/training_session_test_utils.h @@ -18,8 +18,6 @@ #ifdef USE_CUDA #include "core/providers/cuda/cuda_execution_provider_info.h" -#elif USE_ROCM -#include "core/providers/rocm/rocm_execution_provider_info.h" #endif namespace onnxruntime { diff --git a/orttraining/orttraining/test/training_ops/cpu/math/isfinite_ops_test.cc b/orttraining/orttraining/test/training_ops/cpu/math/isfinite_ops_test.cc index d8d2cb8e83550..8caaeeda76936 100644 --- a/orttraining/orttraining/test/training_ops/cpu/math/isfinite_ops_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/math/isfinite_ops_test.cc @@ -10,7 +10,7 @@ namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(IsFiniteTest, Float) { OpTester test("IsFinite", 1, kMSDomain); @@ -256,4 +256,4 @@ TEST(IsAllFiniteTest, MoreFalseFloatTensorLargeFloat16) { #endif } // namespace test -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc b/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc index 60c3ecbcce8ce..edbcb54fc5261 100644 --- a/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/reduction/reduction_ops_test.cc @@ -11,7 +11,7 @@ namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) void test_all_1d_true(size_t size) { std::unique_ptr p_data(new bool[size]); @@ -100,7 +100,7 @@ TEST_P(ReductionOpTest, ReduceAllL2) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST_P(ReductionOpTest, ReduceAllL2HalfHalf) { OpTester test("ReduceAllL2", 1, onnxruntime::kMSDomain, true); test.SetDeterminism(GetParam()); @@ -164,7 +164,7 @@ TEST_P(ReductionOpTest, ReduceAllL2HalfFloat) { } #endif -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST_P(ReductionOpTest, ReduceAllL2_BFloat16_BFloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -193,8 +193,6 @@ TEST_P(ReductionOpTest, ReduceAllL2_BFloat16_BFloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -226,8 +224,6 @@ TEST_P(ReductionOpTest, ReduceAllL2_BFloat16_Float) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -257,8 +253,6 @@ TEST_P(ReductionOpTest, ReduceAllL2_Float_BFloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -441,7 +435,7 @@ TEST(ReductionOpTest, ReduceSumTraining_neg_axis) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(ReductionOpTest, ReduceSumTrainingHalfHalf) { OpTester test("ReduceSumTraining", 1, onnxruntime::kMSDomain); test.AddAttribute("keepdims", (int64_t)0); diff --git a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc index ced03d9df5c29..24c6d69efbcb4 100644 --- a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc @@ -96,7 +96,7 @@ void RunGatherGradTestWithRandomData( } } // namespace -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) // TODO: Currently this cannot pass CI, due to GPU architecture problem TEST(GatherOpTest, Gather_axis0_indices2d_half) { #ifdef USE_CUDA @@ -186,7 +186,7 @@ TEST(GatherGradOpTest, GatherFewDistinctIndices) { RunGatherGradTestWithRandomData(0, {2, 32}, {6, 128}, absolute_error); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) namespace { void RunGatherGradConsistentOutputTest( int64_t axis, diff --git a/orttraining/orttraining/test/training_ops/cuda/activations_test.cc b/orttraining/orttraining/test/training_ops/cuda/activations_test.cc index 3173610597f71..a974c2c8e2884 100644 --- a/orttraining/orttraining/test/training_ops/cuda/activations_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/activations_test.cc @@ -8,8 +8,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif static void TestActivations(const std::vector& tensor_dim, diff --git a/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc b/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc index d842d4f1ea736..e99b700beeba5 100644 --- a/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc @@ -14,7 +14,7 @@ namespace test { using namespace onnxruntime::test; -#if USE_CUDA || USE_ROCM +#if USE_CUDA static void TestBatchNormInternal(bool test_double = false, bool T_is_half = false, bool T1_is_half = false, bool T2_is_half = false, const std::vector& input_output_dims = {2, 2, 2, 2}) { @@ -137,11 +137,9 @@ TEST(CudaKernelTest, BNInternalBasic) { // float case TestBatchNormInternal(); } -#ifndef USE_ROCM // MIOpen does not support double type TEST(CudaKernelTest, BNInternalDouble) { // double case TestBatchNormInternal(true); } -#endif // ndef USE_ROCM TEST(CudaKernelTest, BNInternalHalf) { // half case TestBatchNormInternal(false, true, true, true); @@ -196,7 +194,7 @@ TEST(CudaKernelTest, BNInternal1DInput) { // float case, 1d input test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCpuExecutionProvider, kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } -#endif // USE_CUDA || USE_ROCM +#endif // USE_CUDA } // namespace test } // namespace contrib diff --git a/orttraining/orttraining/test/training_ops/cuda/batch_scale_test.cc b/orttraining/orttraining/test/training_ops/cuda/batch_scale_test.cc index eb229b82caa55..4700589de8d57 100644 --- a/orttraining/orttraining/test/training_ops/cuda/batch_scale_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/batch_scale_test.cc @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" diff --git a/orttraining/orttraining/test/training_ops/cuda/bitmask_dropout_grad_test.cc b/orttraining/orttraining/test/training_ops/cuda/bitmask_dropout_grad_test.cc index 434d1804931b0..3ede24ac9b1fe 100644 --- a/orttraining/orttraining/test/training_ops/cuda/bitmask_dropout_grad_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/bitmask_dropout_grad_test.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include #include @@ -10,23 +10,14 @@ #include "test/providers/provider_test_utils.h" #include "test/common/tensor_op_test_utils.h" #include "test/util/include/default_providers.h" -#ifdef USE_ROCM -#include "core/providers/rocm/shared_inc/rocm_utils.h" -#else #include "core/providers/cuda/shared_inc/cuda_utils.h" -#endif namespace onnxruntime { namespace contrib { namespace test { -#ifdef USE_ROCM -using onnxruntime::rocm::BitmaskElementType; -using onnxruntime::rocm::kNumBitsPerBitmaskElement; -#else using onnxruntime::cuda::BitmaskElementType; using onnxruntime::cuda::kNumBitsPerBitmaskElement; -#endif namespace { @@ -85,8 +76,6 @@ void RunTest(const std::vector& input_dims) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(onnxruntime::test::DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(onnxruntime::test::DefaultRocmExecutionProvider()); #endif test.Run(onnxruntime::test::OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/orttraining/orttraining/test/training_ops/cuda/conv_grad_test.cc b/orttraining/orttraining/test/training_ops/cuda/conv_grad_test.cc index 691856c688c9f..7638596bc3997 100644 --- a/orttraining/orttraining/test/training_ops/cuda/conv_grad_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/conv_grad_test.cc @@ -11,7 +11,7 @@ namespace test { using namespace std; using namespace onnxruntime::test; -#if USE_CUDA || USE_ROCM +#if USE_CUDA namespace { struct ConvGradOpAttributes { @@ -315,7 +315,7 @@ TEST(ConvTest, Conv3D_Bias) { TestConvGradOp(attrs, {dY, X, W}, {dY_shape, X_shape, W_shape}, {dX, dW, dB}, {dX_shape, dW_shape, dB_shape}); TestConvGradOp(attrs, {dY, X, W}, {dY_shape, X_shape, W_shape}, {dX, dW, dB}, {dX_shape, dW_shape, dB_shape}, true); } -#endif // USE_CUDA || USE_ROCM +#endif // USE_CUDA } // namespace test } // namespace contrib diff --git a/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc b/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc index 61bd9c19f3541..9dd0e59438be9 100644 --- a/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc @@ -15,8 +15,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif static void TestSoftmaxCrossEntropy(const std::vector& X_dims, @@ -423,8 +421,6 @@ static void TestSCELoss(const char* op, int opset_version, []() -> std::unique_ptr { #ifdef USE_CUDA return DefaultCudaExecutionProvider(); -#elif USE_ROCM - return DefaultRocmExecutionProvider(); #endif }, reduction, ignore_index, @@ -934,8 +930,6 @@ static void TestSoftmaxCrossEntropyLossInternalGrad(const std::vector& []() -> std::unique_ptr { #ifdef USE_CUDA return DefaultCudaExecutionProvider(); -#elif USE_ROCM - return DefaultRocmExecutionProvider(); #endif }, reduction, ignore_index, error_tolerance, has_bias, diff --git a/orttraining/orttraining/test/training_ops/cuda/flatten_and_unpad_test.cc b/orttraining/orttraining/test/training_ops/cuda/flatten_and_unpad_test.cc index dd5fa18ab3edd..ab87420d83dfb 100644 --- a/orttraining/orttraining/test/training_ops/cuda/flatten_and_unpad_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/flatten_and_unpad_test.cc @@ -7,7 +7,7 @@ namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(FlattenAndUnpadTest, Int32Type2D) { std::vector input = {1, 1, 3, 2, 0, 3, 0, 4, diff --git a/orttraining/orttraining/test/training_ops/cuda/gather_elements_grad_test.cc b/orttraining/orttraining/test/training_ops/cuda/gather_elements_grad_test.cc index 07d407da8e8d2..f1b545bda92ab 100644 --- a/orttraining/orttraining/test/training_ops/cuda/gather_elements_grad_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/gather_elements_grad_test.cc @@ -8,7 +8,7 @@ #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) #include "test/providers/kernel_compute_test_utils.h" #endif @@ -142,7 +142,7 @@ void RunTestWrapper() { RunTest({2, 1, 1, 2, 3, 2, 3}, {2, 1, 1, 2, 3, 2, 2}, true, -5LL); } -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) template void RunKernelComputeTest(std::initializer_list input_dims, std::initializer_list indices_dims, std::initializer_list indices_strides = {}, bool has_axis = false, @@ -154,8 +154,6 @@ void RunKernelComputeTest(std::initializer_list input_dims, std::initia GetData(input_dims, indices_dims, indices_strides, new_axis, dY_data, indices_data, dX_data); #ifdef USE_CUDA const char* provider = kCudaExecutionProvider; -#else // USE_ROCM - const char* provider = kRocmExecutionProvider; #endif onnxruntime::test::KernelComputeTester test("GatherElementsGrad", provider, 1, kMSDomain); if (has_axis) test.AddAttribute("axis", axis); @@ -193,7 +191,7 @@ TEST(GatherElementsGrad, double) { RunTestWrapper(); } TEST(GatherElementsGrad, MLFloat16) { RunTestWrapper(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(GatherElementsGrad, BFloat16) { #ifdef USE_CUDA @@ -219,7 +217,7 @@ TEST(GatherElementsGrad, IndicesUpdatesDontMatch) { test.Run(onnxruntime::test::OpTester::ExpectResult::kExpectFailure, ""); } -#if defined(ENABLE_STRIDED_TENSORS) && (defined(USE_CUDA) || defined(USE_ROCM)) +#if defined(ENABLE_STRIDED_TENSORS) && defined(USE_CUDA) TEST(GatherElementsGrad, Strided_float) { RunKernelComputeTestWrapper(); } TEST(GatherElementsGrad, Strided_double) { RunKernelComputeTestWrapper(); } diff --git a/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc b/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc index 13ad2f6150acf..13ec8cdbe343c 100644 --- a/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc @@ -10,8 +10,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif constexpr auto k_epsilon_default = 1e-5f; diff --git a/orttraining/orttraining/test/training_ops/cuda/mixed_precision_scale_test.cc b/orttraining/orttraining/test/training_ops/cuda/mixed_precision_scale_test.cc index 35b7e8d91d164..03101b9549269 100644 --- a/orttraining/orttraining/test/training_ops/cuda/mixed_precision_scale_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/mixed_precision_scale_test.cc @@ -147,7 +147,7 @@ TEST(CudaKernelTest, MixedPrecisionScaleH2H) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(CudaKernelTest, MixedPrecisionScale_bfloat16_bfloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -166,8 +166,6 @@ TEST(CudaKernelTest, MixedPrecisionScale_bfloat16_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -190,8 +188,6 @@ TEST(CudaKernelTest, MixedPrecisionScale_float_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -214,8 +210,6 @@ TEST(CudaKernelTest, MixedPrecisionScale_bfloat16_float) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -238,8 +232,6 @@ TEST(CudaKernelTest, MixedPrecisionScale_half_bfloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -262,12 +254,10 @@ TEST(CudaKernelTest, MixedPrecisionScale_bfloat16_half) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } #endif } // namespace test -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/orttraining/orttraining/test/training_ops/cuda/negativeloglikelihood_test.cc b/orttraining/orttraining/test/training_ops/cuda/negativeloglikelihood_test.cc index c13ec612135a8..2d78575f6de62 100644 --- a/orttraining/orttraining/test/training_ops/cuda/negativeloglikelihood_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/negativeloglikelihood_test.cc @@ -10,8 +10,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif static void TestNegativeLogLikelihoodLoss(CompareOpTester& test, const std::vector* X_dims, diff --git a/orttraining/orttraining/test/training_ops/cuda/pad_and_unflatten_test.cc b/orttraining/orttraining/test/training_ops/cuda/pad_and_unflatten_test.cc index 9a86955e09379..1b179c2eb35c6 100644 --- a/orttraining/orttraining/test/training_ops/cuda/pad_and_unflatten_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/pad_and_unflatten_test.cc @@ -7,7 +7,7 @@ namespace onnxruntime { namespace test { -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(PadAndUnflattenTest, FloatType1D) { std::vector input = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.f}; diff --git a/orttraining/orttraining/test/training_ops/cuda/reduce_sum_test.cc b/orttraining/orttraining/test/training_ops/cuda/reduce_sum_test.cc index 335e6295fbd7b..23b92e13af19b 100644 --- a/orttraining/orttraining/test/training_ops/cuda/reduce_sum_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/reduce_sum_test.cc @@ -8,8 +8,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif static void TestReduceSum(const std::vector& X_dims, diff --git a/orttraining/orttraining/test/training_ops/cuda/resize_grad_test.cc b/orttraining/orttraining/test/training_ops/cuda/resize_grad_test.cc index 8fc13af8816be..f28cc1fda4c47 100644 --- a/orttraining/orttraining/test/training_ops/cuda/resize_grad_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/resize_grad_test.cc @@ -7,7 +7,7 @@ namespace onnxruntime::test { -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) namespace { @@ -22,8 +22,6 @@ TEST(ResizeGradTest, ResizeGradWithSizes) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -51,8 +49,6 @@ TEST(ResizeGradTest, ResizeGradWithSizesHalf) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -86,8 +82,6 @@ TEST(ResizeGradTest, ResizeGradWithSizesAndAlignCorners) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -118,8 +112,6 @@ TEST(ResizeGradTest, ResizeGradWithScales) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -152,8 +144,6 @@ TEST(ResizeGradTest, ResizeGradWithScalesHalf) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -192,8 +182,6 @@ TEST(ResizeGradTest, ResizeGradWithScalesAndAlignCorners) { std::vector> providers; #ifdef USE_CUDA providers.emplace_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - providers.emplace_back(DefaultRocmExecutionProvider()); #endif OpTester test("ResizeGrad", 1, onnxruntime::kMSDomain); @@ -222,6 +210,6 @@ TEST(ResizeGradTest, ResizeGradWithScalesAndAlignCorners) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &providers); } -#endif // defined(USE_CUDA) || defined(USE_ROCM) +#endif // defined(USE_CUDA) } // namespace onnxruntime::test diff --git a/orttraining/orttraining/test/training_ops/cuda/scale_test.cc b/orttraining/orttraining/test/training_ops/cuda/scale_test.cc index ec48cccf927b6..64875ec18835c 100644 --- a/orttraining/orttraining/test/training_ops/cuda/scale_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/scale_test.cc @@ -134,7 +134,7 @@ TEST(CudaKernelTest, ScaleHalfInt64ScaleDown) { test.Run(); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) TEST(CudaKernelTest, ScaleBFloat16BFloat16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -152,8 +152,6 @@ TEST(CudaKernelTest, ScaleBFloat16BFloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -175,8 +173,6 @@ TEST(CudaKernelTest, ScaleFloatBFloat16) { std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/orttraining/orttraining/test/training_ops/cuda/scaled_sum_test.cc b/orttraining/orttraining/test/training_ops/cuda/scaled_sum_test.cc index ae55aaa1afb6b..ef6d8b9f46e3a 100644 --- a/orttraining/orttraining/test/training_ops/cuda/scaled_sum_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/scaled_sum_test.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" diff --git a/orttraining/orttraining/test/training_ops/cuda/softmax_dropout_test.cc b/orttraining/orttraining/test/training_ops/cuda/softmax_dropout_test.cc index 8c9ff298cad9c..1f81c6bed233c 100644 --- a/orttraining/orttraining/test/training_ops/cuda/softmax_dropout_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/softmax_dropout_test.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) #include #include @@ -74,8 +74,6 @@ void LaunchBiasSoftmaxDropoutTester(const std::vector& input_dims, cons std::vector> eps; #ifdef USE_CUDA eps.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - eps.push_back(DefaultRocmExecutionProvider()); #endif tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &eps); } @@ -175,8 +173,6 @@ void LaunchSoftmaxDropoutGradTester(const std::vector& dims, const std: std::vector> execution_providers; #ifdef USE_CUDA execution_providers.push_back(DefaultCudaExecutionProvider()); -#elif USE_ROCM - execution_providers.push_back(DefaultRocmExecutionProvider()); #endif test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } diff --git a/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc b/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc index 9ced022aab850..9a9467e74b506 100644 --- a/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc @@ -8,8 +8,6 @@ namespace test { #if USE_CUDA constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider; -#elif USE_ROCM -constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider; #endif template @@ -215,22 +213,14 @@ TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_LastAxis_Float16) { std::vector dY_dims{8, 16, 2048}; std::vector Y_dims{8, 16, 2048}; std::vector dX_dims{8, 16, 2048}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, false, 1.5e-2, 1.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, false, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo) { std::vector dY_dims{8, 16, 1500}; std::vector Y_dims{8, 16, 1500}; std::vector dX_dims{8, 16, 1500}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, false, 1.7e-2, 1.7e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, false, 1e-3, 1e-3); -#endif } // large tensor to check cuda DNN softmax backward @@ -246,26 +236,16 @@ TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_AllAxis_Float16) { std::vector dY_dims{8, 16, 512}; std::vector Y_dims{8, 16, 512}; std::vector dX_dims{8, 16, 512}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, false, 1.5e-2, 1.5e-2); - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, false, 1.5e-2, 1.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, false, 1e-3, 1e-3); TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, false, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo) { std::vector dY_dims{8, 16, 1500}; std::vector Y_dims{8, 16, 1500}; std::vector dX_dims{8, 16, 1500}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, false, 2.5e-2, 2.5e-2); - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, false, 2.5e-2, 2.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, false, 1e-3, 1e-3); TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, false, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, LogSoftmaxGrad_SmallTensor_LastAxis) { @@ -294,23 +274,14 @@ TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_LastAxis_Float16) { std::vector dY_dims{8, 16, 2048}; std::vector Y_dims{8, 16, 2048}; std::vector dX_dims{8, 16, 2048}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true, 3.5e-2, 3.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo) { std::vector dY_dims{8, 16, 1500}; std::vector Y_dims{8, 16, 1500}; std::vector dX_dims{8, 16, 1500}; -#if USE_ROCM - // FIXME: Excessive numerical errors - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true, 1.0, 5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis) { @@ -325,26 +296,16 @@ TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis_Float16) { std::vector dY_dims{8, 16, 512}; std::vector Y_dims{8, 16, 512}; std::vector dX_dims{8, 16, 512}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true, 1.5e-2, 1.5e-2); - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true, 1.5e-2, 1.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true, 1e-3, 1e-3); TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true, 1e-3, 1e-3); -#endif } TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo) { std::vector dY_dims{8, 16, 1500}; std::vector Y_dims{8, 16, 1500}; std::vector dX_dims{8, 16, 1500}; -#if USE_ROCM - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true, 4.5e-2, 4.5e-2); - TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true, 4.5e-2, 4.5e-2); -#else TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true, 1e-3, 1e-3); TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true, 1e-3, 1e-3); -#endif } static void TestSoftmaxGrad_13(const std::vector& dY_dims, diff --git a/orttraining/orttraining/training_api/module.cc b/orttraining/orttraining/training_api/module.cc index 9e12fdcd2bb53..13f713da65eda 100644 --- a/orttraining/orttraining/training_api/module.cc +++ b/orttraining/orttraining/training_api/module.cc @@ -266,7 +266,7 @@ Status Parameter::ResetGrad() { if (device.Type() == OrtDevice::CPU) { memset(p_tensor->MutableDataRaw(), 0, p_tensor->SizeInBytes()); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) else if (device.Type() == OrtDevice::GPU) { ORT_NOT_IMPLEMENTED("Not implemented."); } diff --git a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu index 1963fe0185211..314442cca2e51 100644 --- a/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/activation/bias_gelu_grad_impl.cu @@ -97,12 +97,7 @@ void LaunchBiasGeluGradDxKernel( const int num_elements_per_thread = GridDim::maxElementsPerThread; -#ifdef USE_ROCM - // Optimization for ROCm MI100 - const int max_threads_per_block = 512; -#else const int max_threads_per_block = GridDim::maxThreadsPerBlock; -#endif int num_threads_per_block = std::min(static_cast(CeilDiv(bias_size, num_elements_per_thread)), max_threads_per_block); diff --git a/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout.cc b/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout.cc index 59d2081335d50..147373e77655e 100644 --- a/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout.cc +++ b/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout.cc @@ -39,11 +39,7 @@ struct DispatchBiasSoftmaxDropoutImpl { } // namespace -#ifdef USE_ROCM -#define BIAS_SOFTMAX_DROPOUT_TYPES float, MLFloat16 -#else #define BIAS_SOFTMAX_DROPOUT_TYPES float, MLFloat16, double -#endif ONNX_OPERATOR_KERNEL_EX(BiasSoftmaxDropout, kMSDomain, 1, kCudaExecutionProvider, (*KernelDefBuilder::Create()) diff --git a/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout_impl.cu b/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout_impl.cu index 72fbbf53bfb21..0c144dab8ca20 100644 --- a/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/math/bias_softmax_dropout_impl.cu @@ -21,11 +21,7 @@ __global__ void BiasSoftmaxDropoutKernel(T* dropout_output_data, bool* mask_data constexpr int kNextPowOfTwo = 1 << Log2Elements; constexpr int kWarpSize = kNextPowOfTwo < GPU_WARP_SIZE ? kNextPowOfTwo : GPU_WARP_SIZE; constexpr int kWarpIterations = kNextPowOfTwo / kWarpSize; -#ifdef USE_ROCM - constexpr int kWarpBatch = 1; -#else constexpr int kWarpBatch = (kNextPowOfTwo <= 128) ? 2 : 1; -#endif int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * kWarpBatch; // last warp may have fewer batches. @@ -201,13 +197,8 @@ Status BiasSoftmaxDropoutImpl(cudaStream_t stream, const cudaDeviceProp& prop, c int warp_size = std::min(next_power_of_two, GPU_WARP_SIZE_HOST); // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_forward. -#ifdef USE_ROCM - int batches_per_warp = 1; - constexpr int threads_per_block = 256; -#else int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; constexpr int threads_per_block = 128; -#endif constexpr int t_vec4_alignment = std::alignment_of>::value; constexpr int mask_vec4_alignment = std::alignment_of>::value; diff --git a/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad.cc b/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad.cc index 7399d2e1e933a..51960b3540067 100644 --- a/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad.cc @@ -37,11 +37,7 @@ struct DispatchSoftmaxDropoutGradImpl { } // namespace -#ifdef USE_ROCM -#define SOFTMAX_DROPOUT_GRAD_TYPES float, MLFloat16 -#else #define SOFTMAX_DROPOUT_GRAD_TYPES float, MLFloat16, double -#endif ONNX_OPERATOR_KERNEL_EX(SoftmaxDropoutGrad, kMSDomain, 1, kCudaExecutionProvider, (*KernelDefBuilder::Create()) diff --git a/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad_impl.cu index b48ab1b718786..60a15d1386f4d 100644 --- a/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/math/softmax_dropout_grad_impl.cu @@ -19,11 +19,7 @@ __global__ void SoftmaxDropoutGradKernel(T* input_grad_data, const T* output_gra constexpr int kNextPowOfTwo = 1 << Log2Elements; constexpr int kWarpSize = kNextPowOfTwo < GPU_WARP_SIZE ? kNextPowOfTwo : GPU_WARP_SIZE; constexpr int kWarpIterations = kNextPowOfTwo / kWarpSize; -#ifdef USE_ROCM - constexpr int kWarpBatch = 1; -#else constexpr int kWarpBatch = (kNextPowOfTwo <= 128) ? 2 : 1; -#endif int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * kWarpBatch; // last warp may have fewer batches. @@ -146,13 +142,8 @@ Status SoftmaxDropoutGradImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T int warp_size = std::min(next_power_of_two, GPU_WARP_SIZE_HOST); // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_backward. -#ifdef USE_ROCM - int batches_per_warp = 1; - constexpr int threads_per_block = 256; -#else int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; constexpr int threads_per_block = 128; -#endif constexpr int t_vec4_alignment = std::alignment_of>::value; constexpr int mask_vec4_alignment = std::alignment_of>::value; diff --git a/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc b/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc index 5c73a25fb4c9a..1be96da593cd5 100644 --- a/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/softmax_grad.cc @@ -30,12 +30,7 @@ struct DispatchSoftmaxGradImpl { } // namespace -// MIOpen doesn't support double so ROCm kernel doesn't have double support for now. -#ifdef USE_ROCM -#define SOFTMAX_GRAD_TYPES float, MLFloat16, BFloat16 -#else #define SOFTMAX_GRAD_TYPES float, double, MLFloat16, BFloat16 -#endif #define REGISTER_SOFTMAX_GRAD_KERNEL(name) \ ONNX_OPERATOR_KERNEL_EX( \ diff --git a/orttraining/orttraining/training_ops/cuda/math/softmax_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/math/softmax_grad_impl.cu index 3b5bd895c1f54..0764f60b2d1b5 100644 --- a/orttraining/orttraining/training_ops/cuda/math/softmax_grad_impl.cu +++ b/orttraining/orttraining/training_ops/cuda/math/softmax_grad_impl.cu @@ -36,11 +36,7 @@ __global__ void softmax_warp_backward(output_t* gradInput, const input_t* grad, constexpr int next_power_of_two = 1 << log2_elements; constexpr int WARP_SIZE = (next_power_of_two < GPU_WARP_SIZE) ? next_power_of_two : GPU_WARP_SIZE; constexpr int WARP_ITERATIONS = next_power_of_two / WARP_SIZE; -#ifdef USE_ROCM - constexpr int WARP_BATCH = 1; -#else constexpr int WARP_BATCH = (next_power_of_two <= 128) ? 2 : 1; -#endif int first_batch = (blockDim.y * blockIdx.x + threadIdx.y) * WARP_BATCH; @@ -227,11 +223,7 @@ template Status SoftmaxGradImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* input_grad, const T* output_grad, const T* softmax_output, int element_count, int batch_count, bool is_log_softmax) { if (element_count == 0) return Status::OK(); -#ifdef USE_ROCM - if (element_count <= 1024 && element_count * sizeof(T) <= 4096) { -#else if (element_count <= 2048 && element_count * sizeof(T) <= 4096) { -#endif typedef AccumulationType_t AccT; int log2_elements = log2_ceil(element_count); const int next_power_of_two = 1 << log2_elements; @@ -240,13 +232,8 @@ Status SoftmaxGradImpl(cudaStream_t stream, cudnnHandle_t cudnn_handle, T* input int warp_size = std::min(next_power_of_two, GPU_WARP_SIZE_HOST); // This value must match the WARP_BATCH constexpr value computed inside softmax_warp_backward. -#ifdef USE_ROCM - int batches_per_warp = 1; - constexpr int threads_per_block = 256; -#else int batches_per_warp = (next_power_of_two <= 128) ? 2 : 1; constexpr int threads_per_block = 128; -#endif int warps_per_block = (threads_per_block / warp_size); int batches_per_block = warps_per_block * batches_per_warp; diff --git a/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc b/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc index d5f4303414deb..270fc03e7ff7a 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc +++ b/orttraining/orttraining/training_ops/cuda/nn/layer_norm.cc @@ -80,12 +80,8 @@ Status LayerNormGrad::ComputeInternal(OpKernelContext* p_op bias_grad_data = reinterpret_cast(bias_grad->template MutableData()); } -#ifndef USE_ROCM constexpr int part_size = 16; -#else - // Optimization for ROCm MI100 - constexpr int part_size = 64; -#endif + auto part_grad_gamma = GetScratchBuffer(part_size * n2, p_op_kernel_context->GetComputeStream()); auto part_grad_beta = GetScratchBuffer(part_size * n2, p_op_kernel_context->GetComputeStream()); @@ -135,12 +131,8 @@ Status InvertibleLayerNormGrad::ComputeInternal(OpKernelContext* p_op_k auto scale_grad_data = reinterpret_cast(scale_grad->template MutableData()); auto bias_grad_data = reinterpret_cast(bias_grad->template MutableData()); -#ifndef USE_ROCM constexpr int part_size = 16; -#else - // Optimization for ROCm MI100 - constexpr int part_size = 64; -#endif + auto part_grad_gamma = GetScratchBuffer(part_size * n2, p_op_kernel_context->GetComputeStream()); auto part_grad_beta = GetScratchBuffer(part_size * n2, p_op_kernel_context->GetComputeStream()); diff --git a/orttraining/orttraining/training_ops/cuda/tensor/flatten_and_unpad_impl.h b/orttraining/orttraining/training_ops/cuda/tensor/flatten_and_unpad_impl.h index 75f8c243d3425..bb49ec2743123 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/flatten_and_unpad_impl.h +++ b/orttraining/orttraining/training_ops/cuda/tensor/flatten_and_unpad_impl.h @@ -3,11 +3,7 @@ #pragma once -#ifdef USE_ROCM -#include "core/providers/rocm/shared_inc/rocm_utils.h" -#else #include "core/providers/cuda/shared_inc/cuda_utils.h" -#endif namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/cuda/tensor/pad_and_unflatten_impl.h b/orttraining/orttraining/training_ops/cuda/tensor/pad_and_unflatten_impl.h index 8b015179cebd0..89d145294dfc4 100644 --- a/orttraining/orttraining/training_ops/cuda/tensor/pad_and_unflatten_impl.h +++ b/orttraining/orttraining/training_ops/cuda/tensor/pad_and_unflatten_impl.h @@ -3,11 +3,7 @@ #pragma once -#ifdef USE_ROCM -#include "core/providers/rocm/shared_inc/rocm_utils.h" -#else #include "core/providers/cuda/shared_inc/cuda_utils.h" -#endif namespace onnxruntime { namespace cuda { diff --git a/orttraining/orttraining/training_ops/rocm/activation/gelu_grad_impl_common.cuh b/orttraining/orttraining/training_ops/rocm/activation/gelu_grad_impl_common.cuh deleted file mode 100644 index 2377aae9abb54..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/activation/gelu_grad_impl_common.cuh +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/providers/rocm/cu_inc/common.cuh" -#include "orttraining/training_ops/cpu/activation/gelu_computation_mode.h" - -namespace onnxruntime { -namespace rocm { - -template -__device__ __inline__ T ComputeGeluGradScalar(T dY, T X, gelu_computation_mode::Default) { - const T kAlpha = T(M_2_SQRTPI) * T(M_SQRT1_2) * T(0.5); - return dY * (_Normcdf(X) + X * kAlpha * _Exp(-T(0.5) * X * X)); -} - -template -__device__ __inline__ T ComputeGeluGradScalar(T dY, T X, gelu_computation_mode::Approximation) { - // copied and adapted from DeepSpeed: - // https://github.com/microsoft/DeepSpeed/blob/f5025506de37f617a93eabc2aed7cc4f4bfd7d80/csrc/transformer/gelu_kernels.cu#L10 - - const float X_float = static_cast(X); - - const float sqrt_param = 0.79788456080286535587989211986876f; - const float mul_param = 0.044715f; - - constexpr float one = 1.0; - constexpr float two = 2.0; - - float x2mul = X_float * X_float * mul_param; - - // float tan_h = tanhf(sqrt_param * (X_float + X_float * x2mul)); - float u = two * sqrt_param * (X_float + X_float * x2mul); - float emu = __expf(-u); - float tan_h = two / (one + emu) - one; - - float dg1 = 0.5f * (1.0f + tan_h); - float dg2 = X_float * 0.5f * sqrt_param * (1 - tan_h * tan_h); - float dg3 = dg2 * 3 * x2mul; - - return dY * static_cast(dg1 + dg2 + dg3); -} - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/math/div_grad.cc b/orttraining/orttraining/training_ops/rocm/math/div_grad.cc deleted file mode 100644 index 03669e33e7d2e..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/math/div_grad.cc +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "orttraining/training_ops/rocm/math/div_grad.h" -#include "orttraining/training_ops/rocm/math/div_grad_impl.h" -#include "core/providers/rocm/math/binary_elementwise_ops.h" - -using namespace onnxruntime::common; -namespace onnxruntime { -namespace rocm { - -#define DIVGRAD_REGISTER_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - DivGrad, \ - kMSDomain, \ - 1, \ - T, \ - kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - DivGrad); - -DIVGRAD_REGISTER_KERNEL_TYPED(MLFloat16) -DIVGRAD_REGISTER_KERNEL_TYPED(float) -// DIVGRAD_REGISTER_KERNEL_TYPED(double) - -TensorShapeVector prepended_dimension_1(const TensorShape& shape, size_t total_rank) { - size_t input_rank = shape.NumDimensions(); - if (input_rank == total_rank) - return shape.AsShapeVector(); - - TensorShapeVector dims(total_rank, 1); - - // https://github.com/onnx/onnx/blob/main/docs/Broadcasting.md - // for property 3 of Multidirectional Broadcasting, we need to prepended with a dimension of length 1. - if (input_rank > 0) - std::copy(shape.GetDims().begin(), shape.GetDims().end(), &dims[total_rank - input_rank]); - return dims; -} - -template -Status DivGrad::ComputeInternal(OpKernelContext* context) const { - typedef typename ToHipType::MappedType HipT; - - const Tensor* dy_tensor = context->Input(0); - const Tensor* a_tensor = context->Input(1); - const Tensor* b_tensor = context->Input(2); - const TensorShape& a_shape = a_tensor->Shape(); - const TensorShape& b_shape = b_tensor->Shape(); - const TensorShape& dy_shape = dy_tensor->Shape(); - - // output shapes shall match its corresponding inputs - Tensor* da_output_tensor = context->Output(0, a_shape); - Tensor* db_output_tensor = context->Output(1, b_shape); - if (!da_output_tensor && !db_output_tensor) - return Status::OK(); - - BinaryElementwisePreparation prepare; - ORT_RETURN_IF_ERROR(BinaryElementwiseBroadcastPrepare(a_tensor, b_tensor, - // TODO: BinaryElementwiseBroadcastPrepare shall take dy_tensor as const Tensor*. - const_cast(dy_tensor), &prepare)); - const HipT* prepare_a_data = reinterpret_cast(prepare.lhs_tensor->template Data()); - const HipT* prepare_b_data = reinterpret_cast(prepare.rhs_tensor->template Data()); - const HipT* prepare_dy_data = reinterpret_cast(prepare.output_tensor->template Data()); - T* da_data = da_output_tensor ? da_output_tensor->template MutableData() : nullptr; - T* db_data = db_output_tensor ? db_output_tensor->template MutableData() : nullptr; - - switch (prepare.output_rank_or_simple_broadcast) { - case static_cast(SimpleBroadcast::NoBroadcast): - ImplDivGradSimple( - Stream(context), - SimpleBroadcast::NoBroadcast, - prepare_a_data, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - reinterpret_cast(da_data), - reinterpret_cast(db_data)); - break; - case static_cast(SimpleBroadcast::LeftScalar): { - T* temp_da_data = nullptr; - IAllocatorUniquePtr temp_da_allocator; - if (da_output_tensor) { - temp_da_allocator = GetScratchBuffer(dy_shape.Size(), context->GetComputeStream()); - temp_da_data = temp_da_allocator.get(); - } - - ImplDivGradSimple( - Stream(context), - SimpleBroadcast::LeftScalar, - prepare_a_data, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - reinterpret_cast(temp_da_data), - reinterpret_cast(db_data)); - - if (da_output_tensor) { - auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); - ORT_RETURN_IF_ERROR((ReduceKernelShared( - temp_da_data, - dy_shape, - da_data, - TensorShape({}), - MIOPEN_REDUCE_TENSOR_ADD, - GetMiopenHandle(context), - context->GetComputeStream(), - a_output_dims))); - } - break; - } - case static_cast(SimpleBroadcast::RightScalar): { - T* temp_db_data = nullptr; - IAllocatorUniquePtr temp_db_allocator; - if (db_output_tensor) { - temp_db_allocator = GetScratchBuffer(dy_shape.Size(), context->GetComputeStream()); - temp_db_data = temp_db_allocator.get(); - } - ImplDivGradSimple( - Stream(context), - SimpleBroadcast::RightScalar, - prepare_a_data, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - reinterpret_cast(da_data), - reinterpret_cast(temp_db_data)); - - if (db_output_tensor) { - auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); - ORT_RETURN_IF_ERROR((ReduceKernelShared( - temp_db_data, - dy_shape, - db_data, - TensorShape({}), - MIOPEN_REDUCE_TENSOR_ADD, - GetMiopenHandle(context), - context->GetComputeStream(), - b_output_dims))); - } - break; - } - case static_cast(SimpleBroadcast::RightPerChannelBatch1): - case static_cast(SimpleBroadcast::RightPerChannelBatchN): { - T* temp_db_data = nullptr; - IAllocatorUniquePtr temp_db_allocator; - if (db_output_tensor) { - temp_db_allocator = GetScratchBuffer(dy_shape.Size(), context->GetComputeStream()); - temp_db_data = temp_db_allocator.get(); - } - if (prepare.output_rank_or_simple_broadcast == static_cast(SimpleBroadcast::RightPerChannelBatch1)) { - // lhs(1,C,H) and rhs (C,1) - ImplDivGradRhsPerChannelBatch1( - Stream(context), - prepare_a_data, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - prepare.fdm_H, - reinterpret_cast(da_data), - reinterpret_cast(temp_db_data)); - } else { - // lhs(N,C,H) and rhs (C,1) - ImplDivGradRhsPerChannelBatchN( - Stream(context), - prepare_a_data, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - prepare.fdm_H, - prepare.fdm_C, - reinterpret_cast(da_data), - reinterpret_cast(temp_db_data)); - } - - if (db_output_tensor) { - auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); - ORT_RETURN_IF_ERROR((ReduceKernelShared( - temp_db_data, - dy_shape, - db_data, - b_shape, - MIOPEN_REDUCE_TENSOR_ADD, - GetMiopenHandle(context), - context->GetComputeStream(), - b_output_dims))); - } - break; - } - default: { - bool need_reduce_da = da_output_tensor && a_shape.Size() != dy_shape.Size(); - bool need_reduce_db = db_output_tensor && b_shape.Size() != dy_shape.Size(); - IAllocatorUniquePtr temp_da_allocator, temp_db_allocator; - T* da_data_ref = nullptr; - if (da_output_tensor) { - if (need_reduce_da) { - temp_da_allocator = GetScratchBuffer(dy_shape.Size(), context->GetComputeStream()); - da_data_ref = temp_da_allocator.get(); - } else { - da_data_ref = da_data; - } - } - T* db_data_ref = nullptr; - if (db_output_tensor) { - if (need_reduce_db) { - temp_db_allocator = GetScratchBuffer(dy_shape.Size(), context->GetComputeStream()); - db_data_ref = temp_db_allocator.get(); - } else { - db_data_ref = db_data; - } - } - ImplDivGrad( - Stream(context), - prepare.output_rank_or_simple_broadcast, - prepare.lhs_padded_strides, - prepare_a_data, - prepare.rhs_padded_strides, - prepare_b_data, - prepare_dy_data, - dy_shape.Size(), - prepare.fdm_output_strides, - reinterpret_cast(da_data_ref), - reinterpret_cast(db_data_ref)); - - if (need_reduce_da) { - auto a_output_dims = prepended_dimension_1(a_shape, dy_shape.NumDimensions()); - ORT_RETURN_IF_ERROR((ReduceKernelShared( - da_data_ref, - dy_shape, - da_data, - a_shape, - MIOPEN_REDUCE_TENSOR_ADD, - GetMiopenHandle(context), - context->GetComputeStream(), - a_output_dims))); - } - - if (need_reduce_db) { - auto b_output_dims = prepended_dimension_1(b_shape, dy_shape.NumDimensions()); - ORT_RETURN_IF_ERROR((ReduceKernelShared( - db_data_ref, - dy_shape, - db_data, - b_shape, - MIOPEN_REDUCE_TENSOR_ADD, - GetMiopenHandle(context), - context->GetComputeStream(), - b_output_dims))); - } - } - } - return Status::OK(); -} - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.cc b/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.cc deleted file mode 100644 index b1072796bb4fa..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.cc +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "orttraining/training_ops/rocm/nn/batch_norm_grad.h" -#include "core/providers/common.h" -#include "core/providers/rocm/miopen_common.h" -#include "core/providers/cpu/nn/batch_norm_helper.h" -#include "core/providers/rocm/math/unary_elementwise_ops_impl.h" - -using namespace std; -namespace onnxruntime { -namespace rocm { - -#define REGISTER_GRADIENT_KERNEL_TYPED(T, T1, T2) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - BatchNormalizationGrad, \ - kMSDomain, \ - 1, \ - T##_##T1##_##T2, \ - kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()).TypeConstraint("T1", DataTypeImpl::GetTensorType()).TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ - BatchNormalizationGrad); - -template -Status BatchNormalizationGrad::ComputeInternal(OpKernelContext* ctx) const { - typedef typename ToHipType::MappedType HipT; - typedef typename ToHipType::MappedType HipT1; - typedef typename ToHipType::MappedType HipT2; - - const Tensor* dY = ctx->Input(0); - const Tensor* X = ctx->Input(1); - const Tensor* Scale = ctx->Input(2); - const Tensor* saved_mean = ctx->Input(3); - // miopenBatchNormalizationBackward() claims to use `savedInvVariance`, but the value - // is actually equal to the batch inv_std, so we use name `saved_inv_std` here. - const Tensor* saved_inv_std = ctx->Input(4); - const TensorShape input_shape = X->Shape(); - const TensorShape channel_shape = saved_mean->Shape(); - - // no B here, but B has same size as Scale, so can validate inputs for gradient with this substitute - ORT_RETURN_IF_ERROR(BatchNormHelper::ValidateInputs(X, Scale, Scale, saved_mean, saved_inv_std)); - - auto dY_data = reinterpret_cast(dY->template Data()); - auto X_data = reinterpret_cast(X->template Data()); - auto Scale_data = reinterpret_cast(Scale->template Data()); - auto saved_mean_data = reinterpret_cast(saved_mean->template Data()); - auto saved_inv_std_data = reinterpret_cast(saved_inv_std->template Data()); - - auto dX_data = reinterpret_cast(ctx->Output(0, input_shape)->template MutableData()); - auto dScale_data = reinterpret_cast(ctx->Output(1, channel_shape)->template MutableData()); - auto dBias_data = reinterpret_cast(ctx->Output(2, channel_shape)->template MutableData()); - - const auto alpha = Consts::One; - const auto beta = Consts::Zero; - - MiopenTensor input_tensor, scale_bias_tensor; - vector new_dims; - BatchNormHelper::NormalizeDims(input_shape, new_dims); - ORT_RETURN_IF_ERROR(input_tensor.Set(new_dims, MiopenTensor::GetDataType())); - // for fp16 input, `scale_bias_tensor` will have a float type; otherwise it will be the same as input type. - ORT_RETURN_IF_ERROR(scale_bias_tensor.Set(input_tensor, miopen_batch_norm_mode_)); - - const int64_t C = new_dims[1]; - auto p_scale = reinterpret_cast(Scale_data); - auto p_saved_mean = reinterpret_cast(saved_mean_data); - auto p_saved_inv_std = reinterpret_cast(saved_inv_std_data); - auto p_dScale = reinterpret_cast(dScale_data); - auto p_dBias = reinterpret_cast(dBias_data); - - IAllocatorUniquePtr p_f_scale, p_f_dScale, p_f_dBias, p_f_saved_mean, p_f_saved_inv_std; - - if (std::is_same::value) { - p_f_scale = GetScratchBuffer(C, ctx->GetComputeStream()); - p_f_dScale = GetScratchBuffer(C, ctx->GetComputeStream()); - p_f_dBias = GetScratchBuffer(C, ctx->GetComputeStream()); - - Impl_Cast(Stream(ctx), Scale_data, p_f_scale.get(), C); - - p_scale = p_f_scale.get(); - p_dScale = p_f_dScale.get(); - p_dBias = p_f_dBias.get(); - } - - if (std::is_same::value) { - p_f_saved_mean = GetScratchBuffer(C, ctx->GetComputeStream()); - p_f_saved_inv_std = GetScratchBuffer(C, ctx->GetComputeStream()); - - Impl_Cast(Stream(ctx), saved_mean_data, p_f_saved_mean.get(), C); - Impl_Cast(Stream(ctx), saved_inv_std_data, p_f_saved_inv_std.get(), C); - - p_saved_mean = p_f_saved_mean.get(); - p_saved_inv_std = p_f_saved_inv_std.get(); - } - - MIOPEN_RETURN_IF_ERROR(miopenBatchNormalizationBackward( - GetMiopenHandle(ctx), - miopen_batch_norm_mode_, - &alpha, - &beta, - &alpha, - &beta, - input_tensor, - X_data, - input_tensor, - dY_data, - input_tensor, - dX_data, - scale_bias_tensor, - p_scale, - p_dScale, - p_dBias, - epsilon_, - p_saved_mean, - p_saved_inv_std)); - - if (std::is_same::value) { - Impl_Cast(Stream(ctx), reinterpret_cast(p_dScale), dScale_data, C); - Impl_Cast(Stream(ctx), reinterpret_cast(p_dBias), dBias_data, C); - } - - return Status::OK(); -} - -#define SPECIALIZED_GRADIENT(T, T1, T2) \ - REGISTER_GRADIENT_KERNEL_TYPED(T, T1, T2) \ - template Status BatchNormalizationGrad::ComputeInternal(OpKernelContext* ctx) const; - -SPECIALIZED_GRADIENT(float, float, float) -// MIOpen kernel does not support double, disable for now. -// SPECIALIZED_GRADIENT(double, double, double) -SPECIALIZED_GRADIENT(MLFloat16, MLFloat16, MLFloat16) -SPECIALIZED_GRADIENT(MLFloat16, MLFloat16, float) -SPECIALIZED_GRADIENT(MLFloat16, float, float) - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.h b/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.h deleted file mode 100644 index 63d2370076bab..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_grad.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/providers/rocm/rocm_kernel.h" -#include "core/providers/rocm/miopen_common.h" - -namespace onnxruntime { -namespace rocm { - -template -class BatchNormalizationGrad final : public RocmKernel { - public: - BatchNormalizationGrad(const OpKernelInfo& info) - : RocmKernel{info}, - miopen_batch_norm_mode_(miopenBNSpatial) { - float tmp_epsilon; - ORT_ENFORCE(info.GetAttr("epsilon", &tmp_epsilon).IsOK()); - epsilon_ = ClampMiopenBatchNormEpsilon(static_cast(tmp_epsilon)); - - // spatial or not - int64_t tmp_spatial; - if (info.GetAttr("spatial", &tmp_spatial).IsOK()) { - spatial_ = tmp_spatial; - } - - if (spatial_ == 0) { - miopen_batch_norm_mode_ = miopenBNPerActivation; - } - } - - Status ComputeInternal(OpKernelContext* context) const override; - - private: - double epsilon_; - int64_t spatial_ = 1; // default as per spec - miopenBatchNormMode_t miopen_batch_norm_mode_; -}; - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.cc b/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.cc deleted file mode 100644 index dbd1f95ddee95..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "orttraining/training_ops/rocm/nn/batch_norm_internal.h" -#include "core/providers/common.h" -#include "core/providers/rocm/miopen_common.h" -#include "core/providers/cpu/nn/batch_norm_helper.h" -#include "core/providers/rocm/math/unary_elementwise_ops_impl.h" - -using namespace std; -namespace onnxruntime { -namespace rocm { - -#define REGISTER_KERNEL_TYPED(T, T1, T2) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - BatchNormInternal, \ - kMSDomain, \ - 1, \ - T##_##T1##_##T2, \ - kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()) \ - .Alias(3, 1) \ - .Alias(4, 2) \ - .TypeConstraint("T", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("T1", DataTypeImpl::GetTensorType()) \ - .TypeConstraint("T2", DataTypeImpl::GetTensorType()), \ - BatchNormInternal); - -template -Status BatchNormInternal::ComputeInternal(OpKernelContext* p_op_kernel_context) const { - typedef typename ToHipType::MappedType HipT; - typedef typename ToHipType::MappedType HipT1; - typedef typename ToHipType::MappedType HipT2; - - const Tensor* X = p_op_kernel_context->Input(0); - const Tensor* scale = p_op_kernel_context->Input(1); - const Tensor* B = p_op_kernel_context->Input(2); - const Tensor* mean = p_op_kernel_context->Input(3); - const Tensor* var = p_op_kernel_context->Input(4); - - ORT_RETURN_IF_ERROR(BatchNormHelper::ValidateInputs(X, scale, B, mean, var, spatial_ == 1)); - - const TensorShape& x_shape = X->Shape(); - const TensorShape& channel_shape = mean->Shape(); - - Tensor* Y = p_op_kernel_context->Output(0, x_shape); - Tensor* running_mean = p_op_kernel_context->Output(1, channel_shape); - Tensor* running_var = p_op_kernel_context->Output(2, channel_shape); - Tensor* saved_mean = p_op_kernel_context->Output(3, channel_shape); - // miopenBatchNormalizationForwardTraining() claims to output `resultSaveInvVariance`, but the value - // is actually equal to the batch inv_std, so we use name `saved_inv_std` here. - Tensor* saved_inv_std = p_op_kernel_context->Output(4, channel_shape); - - auto x_data = reinterpret_cast(X->template Data()); - auto scale_data = reinterpret_cast(scale->template Data()); - auto b_data = reinterpret_cast(B->template Data()); - auto mean_data = reinterpret_cast(mean->template Data()); - auto var_data = reinterpret_cast(var->template Data()); - - auto y_data = reinterpret_cast(Y->template MutableData()); - - // In MIOpenBatchNormForward, alpha and beta are not const. - float alpha = 1.0; - float beta = 0.0; - - MiopenTensor data_desc, bn_tensor_desc; - vector new_dims; - BatchNormHelper::NormalizeDims(x_shape, new_dims); - ORT_RETURN_IF_ERROR(data_desc.Set(new_dims, MiopenTensor::GetDataType())); - // for fp16 input, `bn_tensor_desc` will have a float type; otherwise it will be the same as input type. - ORT_RETURN_IF_ERROR(bn_tensor_desc.Set(data_desc, miopen_batch_norm_mode_)); - - auto running_mean_data = reinterpret_cast(running_mean->template MutableData()); - auto running_var_data = reinterpret_cast(running_var->template MutableData()); - auto saved_mean_data = reinterpret_cast(saved_mean->template MutableData()); - auto saved_inv_std_data = reinterpret_cast(saved_inv_std->template MutableData()); - - auto p_scale = reinterpret_cast(scale_data); - auto p_B = reinterpret_cast(b_data); - auto p_running_mean = reinterpret_cast(running_mean_data); - auto p_running_var = reinterpret_cast(running_var_data); - auto p_saved_mean = reinterpret_cast(saved_mean_data); - auto p_saved_inv_std = reinterpret_cast(saved_inv_std_data); - - const int64_t C = new_dims[1]; - IAllocatorUniquePtr p_f_scale, p_f_B, p_f_running_mean, p_f_running_var, p_f_saved_mean, p_f_saved_inv_std; - - if (std::is_same::value) { - // Convert scale/B to float - p_f_scale = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - p_f_B = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - - Impl_Cast(Stream(p_op_kernel_context), scale_data, p_f_scale.get(), C); - Impl_Cast(Stream(p_op_kernel_context), b_data, p_f_B.get(), C); - - p_scale = p_f_scale.get(); - p_B = p_f_B.get(); - } - - if (std::is_same::value) { - // Convert mean/var to float - p_f_running_mean = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - p_f_running_var = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - p_f_saved_mean = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - p_f_saved_inv_std = GetScratchBuffer(C, p_op_kernel_context->GetComputeStream()); - - Impl_Cast(Stream(p_op_kernel_context), mean_data, p_f_running_mean.get(), C); - Impl_Cast(Stream(p_op_kernel_context), var_data, p_f_running_var.get(), C); - - p_running_mean = p_f_running_mean.get(); - p_running_var = p_f_running_var.get(); - p_saved_mean = p_f_saved_mean.get(); - p_saved_inv_std = p_f_saved_inv_std.get(); - } else if (mean_data != running_mean_data) { - HIP_RETURN_IF_ERROR( - hipMemcpyAsync(running_mean_data, mean_data, C * sizeof(T2), hipMemcpyDeviceToDevice, Stream(p_op_kernel_context))); - HIP_RETURN_IF_ERROR( - hipMemcpyAsync(running_var_data, var_data, C * sizeof(T2), hipMemcpyDeviceToDevice, Stream(p_op_kernel_context))); - } - - // NOTE: in miopenBatchNorm, biased std/var is used when calculating `save_inv_std` and `y`, while - // `running_var` is updated using unbiased `batch_var`: - // running_var = (1 - momentum_) * unbiased_batch_var + momentum_ * running_var - // This is inconsistent with BatchNormalization Onnx spec, which uses population variance (biased). - MIOPEN_RETURN_IF_ERROR(miopenBatchNormalizationForwardTraining( - GetMiopenHandle(p_op_kernel_context), - miopen_batch_norm_mode_, - &alpha, - &beta, - data_desc, - x_data, - data_desc, - y_data, - bn_tensor_desc, - const_cast(p_scale), - const_cast(p_B), - 1.0 - momentum_, - p_running_mean, - p_running_var, - epsilon_, - p_saved_mean, - p_saved_inv_std)); - - if (std::is_same::value) { - Impl_Cast(Stream(p_op_kernel_context), reinterpret_cast(p_running_mean), running_mean_data, C); - Impl_Cast(Stream(p_op_kernel_context), reinterpret_cast(p_running_var), running_var_data, C); - Impl_Cast(Stream(p_op_kernel_context), reinterpret_cast(p_saved_mean), saved_mean_data, C); - Impl_Cast(Stream(p_op_kernel_context), reinterpret_cast(p_saved_inv_std), saved_inv_std_data, C); - } - - return Status::OK(); -} - -#define SPECIALIZED_COMPUTE(T, T1, T2) \ - REGISTER_KERNEL_TYPED(T, T1, T2) \ - template Status BatchNormInternal::ComputeInternal(OpKernelContext* ctx) const; - -SPECIALIZED_COMPUTE(float, float, float) -// MIOpen kernel does not support double, disable for now. -// SPECIALIZED_COMPUTE(double, double, double) -SPECIALIZED_COMPUTE(MLFloat16, MLFloat16, MLFloat16) -SPECIALIZED_COMPUTE(MLFloat16, MLFloat16, float) -SPECIALIZED_COMPUTE(MLFloat16, float, float) - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.h b/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.h deleted file mode 100644 index d65b66120a78c..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/batch_norm_internal.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/providers/rocm/rocm_kernel.h" -#include "core/providers/rocm/miopen_common.h" - -namespace onnxruntime { -namespace rocm { - -template -class BatchNormInternal final : public RocmKernel { - public: - BatchNormInternal(const OpKernelInfo& op_kernel_info) - : RocmKernel{op_kernel_info}, - miopen_batch_norm_mode_(miopenBNSpatial), - momentum_(0.9) { - float tmp_epsilon; - ORT_ENFORCE(op_kernel_info.GetAttr("epsilon", &tmp_epsilon).IsOK()); - epsilon_ = ClampMiopenBatchNormEpsilon(static_cast(tmp_epsilon)); - - // spatial or not - int64_t tmp_spatial; - if (op_kernel_info.GetAttr("spatial", &tmp_spatial).IsOK()) { - spatial_ = tmp_spatial; - } - - if (spatial_ == 0) { - miopen_batch_norm_mode_ = miopenBNPerActivation; - } - - float tmp_momentum; - if (op_kernel_info.GetAttr("momentum", &tmp_momentum).IsOK()) { - momentum_ = static_cast(tmp_momentum); - } - } - - Status ComputeInternal(OpKernelContext* context) const override; - - private: - double epsilon_; - int64_t spatial_ = 1; // default as per spec - miopenBatchNormMode_t miopen_batch_norm_mode_; - double momentum_; -}; - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc deleted file mode 100644 index 3b1ed29cb0240..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.cc +++ /dev/null @@ -1,385 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -// TODO Add exhaustive and default cases for algo. - -#include "orttraining/training_ops/rocm/nn/conv_grad.h" - -#include "core/providers/common.h" -#include "core/providers/rocm/shared_inc/fpgeneric.h" -#include - -namespace onnxruntime { -namespace rocm { - -#define REGISTER_GRADIENT_KERNEL_TYPED(T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX(ConvGrad, kMSDomain, 1, T, kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - ConvGrad); - -REGISTER_GRADIENT_KERNEL_TYPED(float) -// MIOpen double support not currently implemented. -// REGISTER_GRADIENT_KERNEL_TYPED(double) -REGISTER_GRADIENT_KERNEL_TYPED(MLFloat16) - -using T_BwdDataPerf = miopenConvAlgoPerf_t; -using T_BwdDataAlgo = miopenConvBwdDataAlgorithm_t; -using T_BwdFilterPerf = miopenConvAlgoPerf_t; -using T_BwdFilterAlgo = miopenConvBwdWeightsAlgorithm_t; - -miopenStatus_t GetWorkspaceSize(const ConvArgs& args, T_BwdDataAlgo algo, size_t* workspace_size) { - return miopenConvolutionBackwardDataGetWorkSpaceSize(args.handle, args.y_tensor, args.x_tensor, args.conv_desc, - args.w_desc, workspace_size); -} - -miopenStatus_t GetWorkspaceSize(const ConvArgs& args, T_BwdFilterAlgo algo, size_t* workspace_size) { - return miopenConvolutionBackwardWeightsGetWorkSpaceSize(args.handle, args.y_tensor, args.x_tensor, args.conv_desc, - args.w_desc, workspace_size); -} - -template -size_t GetMaxWorkspaceSize(const ConvArgs& args, const T_Algo* algo, int n_algo) { - // Calling hipMemGetInfo is not ideal, but our rocm allocator doesn't have a way to get this info. - size_t free, total; - HIP_CALL_THROW(hipMemGetInfo(&free, &total)); - // Assuming 10% of fragmentation. - free = static_cast(static_cast(free) * 0.9); - size_t max_workspace_size = 0; - for (int i = 0; i < n_algo; i++) { - miopenStatus_t status; - size_t workspace_size; - status = GetWorkspaceSize(args, algo[i], &workspace_size); - if (miopenStatusSuccess != status || workspace_size == 0 || workspace_size < max_workspace_size || - workspace_size > free) - continue; - max_workspace_size = workspace_size; - } - - return max_workspace_size; -} - -template -std::vector GetValidAlgorithms(const T_Perf* perf_results, int n_algo) { - std::vector result; - result.reserve(n_algo); - for (int i = 0; i < n_algo; i++) { - T_Perf perf = perf_results[i]; - result.emplace_back(perf); - } - ORT_ENFORCE(result.size() > 0, "No valid convolution algorithms available in MIOpen"); - return result; -} - -struct ConvParamsHash { - // ConvParams must be a trivial type because we read out its memory contents as char* when hashing. - static_assert(std::is_trivial::value, "ConvParams is not a trivial type"); - size_t operator()(const ConvParams& conv_params) const { - auto ptr = reinterpret_cast(&conv_params); - uint32_t value = 0x811C9DC5; - for (int i = 0; i < static_cast(sizeof(ConvParams)); ++i) { - value ^= ptr[i]; - value *= 0x01000193; - } - return static_cast(value); - } -}; - -struct ConvParamsEqual { - // ConvParams must be a trivial type because we read out its memory contents as char* when hashing. - static_assert(std::is_trivial::value, "ConvParams is not a trivial type"); - bool operator()(const ConvParams& a, const ConvParams& b) const { - auto ptr1 = reinterpret_cast(&a); - auto ptr2 = reinterpret_cast(&b); - return memcmp(ptr1, ptr2, sizeof(ConvParams)) == 0; - } -}; - -template -struct AlgoPerfCache { - mutable std::mutex mutex; - std::unordered_map map; - - bool Find(const ConvParams& params, T_Perf* result) { - std::lock_guard guard(mutex); - auto it = map.find(params); - if (it == map.end()) { - return false; - } - *result = it->second; - return true; - } - - void Insert(const ConvParams& params, const T_Perf& algo_perf) { - std::lock_guard guard(mutex); - map[params] = algo_perf; - } -}; - -// TODO: Currently we use global AlgoPerfCache for ConvGrad only. Conv's perf cache is still per node. -// Need to apply such global cache for Conv, and move some shared code from here to conv.h/cc. -AlgoPerfCache bwd_data_algos; -AlgoPerfCache bwd_filter_algos; - -template -struct AlgoSearch {}; - -template <> -struct AlgoSearch { - static constexpr auto DEFAULT_ALGO = miopenConvolutionBwdDataAlgoGEMM; - static AlgoPerfCache& Cache() { return bwd_data_algos; } - static Status FindAlgorithms(const ConvArgs& args, const ROCMExecutionProvider* provider, const AllocatorPtr& allocator, - std::vector& perf_results) { - static const T_BwdDataAlgo algos[] = { - miopenConvolutionBwdDataAlgoGEMM, - miopenConvolutionBwdDataAlgoDirect, - miopenConvolutionBwdDataAlgoFFT, - miopenConvolutionBwdDataAlgoWinograd, - miopenTransposeBwdDataAlgoGEMM, - miopenConvolutionBwdDataAlgoImplicitGEMM}; - static constexpr int num_algos = MIOPEN_CONVOLUTION_BWD_DATA_ALGO_COUNT; - ORT_ENFORCE(sizeof(algos) / sizeof(algos[0]) == num_algos, "Missing MIOpen convolution backward data algorithms."); - int perf_count; - std::unique_ptr candidates = std::make_unique(num_algos); - size_t max_workspace_size = provider->GetMiopenConvUseMaxWorkspace() ? GetMaxWorkspaceSize(args, algos, num_algos) - : AlgoSearchWorkspaceSize; - // Use GetTransientScratchBuffer() so the workspace can be freed instead of cached. - // Because the benchmarking uses a huge amount of memory, e.g. a few GBs. - IAllocatorUniquePtr workspace = max_workspace_size == 0 ? nullptr : IAllocator::MakeUniquePtr(allocator, max_workspace_size, true); - MIOPEN_RETURN_IF_ERROR(miopenFindConvolutionBackwardDataAlgorithm( - args.handle, args.y_tensor, args.dy_data, args.w_desc, args.w_data, args.conv_desc, args.x_tensor, - args.dx_data, 1, &perf_count, candidates.get(), workspace.get(), max_workspace_size, false)); - perf_results = GetValidAlgorithms(candidates.get(), perf_count); - return Status::OK(); - } -}; - -template <> -struct AlgoSearch { - static constexpr auto DEFAULT_ALGO = miopenConvolutionBwdWeightsAlgoGEMM; - static AlgoPerfCache& Cache() { return bwd_filter_algos; } - static Status FindAlgorithms(const ConvArgs& args, const ROCMExecutionProvider* provider, const AllocatorPtr& allocator, - std::vector& perf_results) { - static const T_BwdFilterAlgo algos[] = { - miopenConvolutionBwdWeightsAlgoGEMM, - miopenConvolutionBwdWeightsAlgoDirect, - miopenConvolutionBwdWeightsAlgoWinograd, - miopenConvolutionBwdWeightsAlgoImplicitGEMM}; - - static constexpr int num_algos = MIOPEN_CONVOLUTION_BWD_FILTER_ALGO_COUNT; - ORT_ENFORCE(sizeof(algos) / sizeof(algos[0]) == num_algos, "Missing MIOpen convolution backward filter algorithms."); - std::unique_ptr candidates = std::make_unique(num_algos); - int perf_count; - size_t max_workspace_size = provider->GetMiopenConvUseMaxWorkspace() ? GetMaxWorkspaceSize(args, algos, num_algos) - : AlgoSearchWorkspaceSize; - // Use GetTransientScratchBuffer() so the workspace can be freed instead of cached. - // Because the benchmarking uses a huge amount of memory, e.g. a few GBs. - IAllocatorUniquePtr workspace = max_workspace_size == 0 ? nullptr : IAllocator::MakeUniquePtr(allocator, max_workspace_size, true); - MIOPEN_RETURN_IF_ERROR(miopenFindConvolutionBackwardWeightsAlgorithm( - args.handle, args.y_tensor, args.dy_data, args.x_tensor, args.x_data, args.conv_desc, args.w_desc, - args.dw_data, 1, &perf_count, candidates.get(), workspace.get(), max_workspace_size, false)); - perf_results = GetValidAlgorithms(candidates.get(), perf_count); - return Status::OK(); - } -}; - -template -class AlgoIterator { - public: - AlgoIterator(const ConvArgs& args) : args_(args) {} - - Status OnlyDefaultAlgorithm(const ConvArgs& args, std::vector& perf_results); - - Status TryAll(const ROCMExecutionProvider* provider, const AllocatorPtr& allocator, std::function f) { - auto& cache = AlgoSearch::Cache(); - miopenConvAlgoPerf_t algo_perf; - if (cache.Find(args_.params, &algo_perf) && f(algo_perf) == Status::OK()) { - return Status::OK(); - } - - std::vector perf_results; - ORT_RETURN_IF_ERROR(AlgoSearch::FindAlgorithms(args_, provider, allocator, perf_results)); - for (auto& algo_perf : perf_results) { - if (f(algo_perf) == Status::OK()) { - cache.Insert(args_.params, algo_perf); - return Status::OK(); - } - } - ORT_ENFORCE(false, "Unable to find a valid MIOpen algorithm to run convolution."); - return Status::OK(); - } - - private: - const ConvArgs& args_; -}; - -template <> -Status AlgoIterator::OnlyDefaultAlgorithm(const ConvArgs& args, std::vector& perf_results) { - perf_results.resize(1); - perf_results[0].bwd_data_algo = AlgoSearch::DEFAULT_ALGO; - MIOPEN_RETURN_IF_ERROR(GetWorkspaceSize(args, perf_results[0].bwd_data_algo, &(perf_results[0].memory))); - return Status::OK(); -} - -template <> -Status AlgoIterator::OnlyDefaultAlgorithm(const ConvArgs& args, std::vector& perf_results) { - perf_results.resize(1); - perf_results[0].bwd_weights_algo = AlgoSearch::DEFAULT_ALGO; - MIOPEN_RETURN_IF_ERROR(GetWorkspaceSize(args, perf_results[0].bwd_weights_algo, &(perf_results[0].memory))); - return Status::OK(); -} - -template -Status ConvGrad::PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& w, Tensor* dB, Tensor* dX, - Tensor* dW, miopenHandle_t miopen_handle) const { - const TensorShape& x_shape = x.Shape(); - auto x_dims = x_shape.AsShapeVector(); - args_.x_data = reinterpret_cast(x.template Data()); - - const TensorShape& dy_shape = dY.Shape(); - auto dy_dims = dy_shape.AsShapeVector(); - args_.dy_data = reinterpret_cast(dY.template Data()); - - const TensorShape& w_shape = w.Shape(); - auto w_dims = w_shape.AsShapeVector(); - args_.w_data = reinterpret_cast(w.template Data()); - - args_.db_data = dB ? reinterpret_cast(dB->template MutableData()) : nullptr; - args_.dx_data = dX ? reinterpret_cast(dX->template MutableData()) : nullptr; - args_.dw_data = dW ? reinterpret_cast(dW->template MutableData()) : nullptr; - - bool x_dims_changed = (args_.last_x_dims != x_dims); - bool w_dims_changed = (args_.last_w_dims != w_dims); - if (x_dims_changed || w_dims_changed) { - if (x_dims_changed) args_.last_x_dims = x_dims; - if (w_dims_changed) args_.last_w_dims = w_dims; - - // Update Attributes - ORT_RETURN_IF_ERROR(conv_attrs_.ValidateInputShape(&x, &w)); - - TensorShapeVector kernel_shape; - ORT_RETURN_IF_ERROR(conv_attrs_.ComputeKernelShape(w_shape, kernel_shape)); - auto rank = kernel_shape.size(); - - ConvAttributes::ConvPadVector pads(conv_attrs_.pads); - if (pads.empty()) { - pads.resize(rank * 2, 0); - } - - TensorShapeVector dilations(conv_attrs_.dilations); - if (dilations.empty()) { - dilations.resize(rank, 1); - } - - TensorShapeVector strides(conv_attrs_.strides); - if (strides.empty()) { - strides.resize(rank, 1); - } - - // MIOpen only takes 4D or 5D x tensor, so pad dimensions if needed. - if (rank < 2) { - x_dims.push_back(1); - dy_dims.push_back(1); - w_dims.push_back(1); - pads.insert(pads.begin() + rank, 0); - pads.insert(pads.end(), 0); - kernel_shape.push_back(1); - strides.push_back(1); - dilations.push_back(1); - } - - const ROCMExecutionProvider* rocm_ep = - static_cast(this->Info().GetExecutionProvider()); - memset(&args_.params, 0, sizeof(ConvParams)); - args_.params.device_id = static_cast(rocm_ep->GetDeviceId()); - args_.params.data_type = MiopenTensor::GetDataType(); - args_.params.input_dim = static_cast(x_dims.size()); - for (size_t i = 0; i < x_dims.size(); i++) { - args_.params.input_size[i] = static_cast(x_dims[i]); - args_.params.weight_size[i] = static_cast(w_dims[i]); - } - for (size_t i = 0; i < rank; i++) { - args_.params.padding[i] = static_cast(pads[i]); - args_.params.padding[i + rank] = static_cast(pads[i + rank]); - args_.params.stride[i] = static_cast(strides[i]); - args_.params.dilation[i] = static_cast(dilations[i]); - } - args_.params.groups = conv_attrs_.group; - args_.handle = miopen_handle; - ORT_RETURN_IF_ERROR(args_.w_desc.Set(w_dims, args_.params.data_type)); - ORT_RETURN_IF_ERROR(args_.x_tensor.Set(x_dims, args_.params.data_type)); - ORT_RETURN_IF_ERROR(args_.y_tensor.Set(dy_dims, args_.params.data_type)); - ORT_RETURN_IF_ERROR(args_.conv_desc.Set(kernel_shape.size(), pads, strides, dilations, - gsl::narrow_cast(conv_attrs_.group), miopenConvolution, - args_.params.data_type)); - - if (dB) { - const TensorShape& db_shape = dB->Shape(); - ORT_RETURN_IF_NOT(db_shape.NumDimensions() == 1, "bias should be 1D"); - TensorShapeVector db_dims(2 + kernel_shape.size(), 1); - db_dims[1] = db_shape[0]; - ORT_RETURN_IF_ERROR(args_.b_tensor.Set(db_dims, MiopenTensor::GetDataType())); - } - } - - return Status::OK(); -} - -template -Status ConvGrad::ComputeInternal(OpKernelContext* context) const { - const Tensor* dY = context->Input(0); - const Tensor* X = context->Input(1); - const Tensor* W = context->Input(2); - Tensor* dX = context->Output(0, X->Shape()); - Tensor* dW = context->Output(1, W->Shape()); - Tensor* dB = context->Output(2, {W->Shape()[0]}); - ORT_RETURN_IF_ERROR(PrepareArgs(*X, *dY, *W, dB, dX, dW, GetMiopenHandle(context))); - if (dX) ORT_RETURN_IF_ERROR(ComputeInputGradient(context->GetComputeStream())); - if (dW) ORT_RETURN_IF_ERROR(ComputeWeightGradient(context->GetComputeStream())); - if (dB) ORT_RETURN_IF_ERROR(ComputeBiasGradient()); - return Status::OK(); -} - -template -Status ConvGrad::ComputeInputGradient(onnxruntime::Stream* stream) const { - return AlgoIterator(args_).TryAll( - static_cast(Info().GetExecutionProvider()), - Info().GetAllocator(OrtMemType::OrtMemTypeDefault), - [&](const T_BwdDataPerf& algo_perf) -> Status { - const auto one = Consts::One; - const auto zero = Consts::Zero; - IAllocatorUniquePtr workspace = GetScratchBuffer(algo_perf.memory, stream); - MIOPEN_RETURN_IF_ERROR(miopenConvolutionBackwardData( - args_.handle, &one, args_.y_tensor, args_.dy_data, args_.w_desc, args_.w_data, args_.conv_desc, - algo_perf.bwd_data_algo, &zero, args_.x_tensor, args_.dx_data, workspace.get(), algo_perf.memory)); - return Status::OK(); - }); -} - -template -Status ConvGrad::ComputeWeightGradient(onnxruntime::Stream* stream) const { - return AlgoIterator(args_).TryAll( - static_cast(Info().GetExecutionProvider()), - Info().GetAllocator(OrtMemType::OrtMemTypeDefault), - [&](const T_BwdFilterPerf& algo_perf) -> Status { - const auto one = Consts::One; - const auto zero = Consts::Zero; - IAllocatorUniquePtr workspace = GetScratchBuffer(algo_perf.memory, stream); - MIOPEN_RETURN_IF_ERROR(miopenConvolutionBackwardWeights( - args_.handle, &one, args_.y_tensor, args_.dy_data, args_.x_tensor, args_.x_data, args_.conv_desc, - algo_perf.bwd_weights_algo, &zero, args_.w_desc, args_.dw_data, workspace.get(), algo_perf.memory)); - return Status::OK(); - }); -} - -template -Status ConvGrad::ComputeBiasGradient() const { - const auto one = Consts::One; - const auto zero = Consts::Zero; - MIOPEN_RETURN_IF_ERROR(miopenConvolutionBackwardBias( - args_.handle, &one, args_.y_tensor, args_.dy_data, &zero, - args_.b_tensor, args_.db_data)); - return Status::OK(); -} - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h b/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h deleted file mode 100644 index d1f84c259a66a..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/nn/conv_grad.h +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -#include "core/providers/rocm/miopen_common.h" -#include "core/providers/cpu/nn/conv_attributes.h" -#include "core/providers/rocm/nn/conv.h" - -namespace onnxruntime { -namespace rocm { - -constexpr int MAX_DIM = 3; - -struct ConvParams { - int8_t device_id; - miopenDataType_t data_type; - int input_size[2 + MAX_DIM]; - uint8_t input_dim; - int weight_size[2 + MAX_DIM]; - int padding[MAX_DIM * 2]; - int stride[MAX_DIM]; - int dilation[MAX_DIM]; - int64_t groups; -}; - -struct ConvArgs { - // Update needed if x or w's dims changed. - TensorShapeVector last_x_dims; - TensorShapeVector last_w_dims; - - miopenHandle_t handle; - ConvParams params; - MiopenTensor x_tensor, y_tensor, b_tensor; - MiopenTensorDescriptor w_desc; - MiopenConvolutionDescriptor conv_desc; - const void* x_data; - const void* w_data; - const void* dy_data; - void* dx_data; - void* dw_data; - void* db_data; -}; - -template -class ConvGrad final : public RocmKernel { - public: - using HipT = typename ToHipType::MappedType; - - ConvGrad(const OpKernelInfo& info) : RocmKernel(info), conv_attrs_(info) { - auto pads_size = conv_attrs_.pads.size(); - ORT_ENFORCE(pads_size % 2 == 0); - } - - Status ComputeInternal(OpKernelContext* context) const override; - - protected: - Status PrepareArgs(const Tensor& x, const Tensor& dY, const Tensor& w, Tensor* dB, Tensor* dX, Tensor* dW, miopenHandle_t miopen_handle) const; - mutable ConvArgs args_; - ConvAttributes conv_attrs_; - - private: - Status ComputeWeightGradient(onnxruntime::Stream* stream) const; - Status ComputeInputGradient(onnxruntime::Stream* stream) const; - Status ComputeBiasGradient() const; -}; - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/reduction/reduction_all.cc b/orttraining/orttraining/training_ops/rocm/reduction/reduction_all.cc deleted file mode 100644 index 093a516ce8241..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/reduction/reduction_all.cc +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "orttraining/training_ops/rocm/reduction/reduction_all.h" -#include "orttraining/training_ops/rocm/reduction/reduction_all_impl.h" - -#include "core/providers/rocm/reduction/reduction_functions.h" -#include "core/providers/rocm/shared_inc/accumulation_type.h" - -namespace onnxruntime { -namespace rocm { - -#define REGISTER_REDUCE_ALL_KERNEL_TYPED(Name, TIn, TOut) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - Name, \ - kMSDomain, \ - 1, \ - TIn##_##TOut, \ - kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()).TypeConstraint("TIn", DataTypeImpl::GetTensorType()).TypeConstraint("TOut", DataTypeImpl::GetTensorType()), \ - Name); - -template -Status ReduceAllL2::ComputeInternal(OpKernelContext* ctx) const { - typedef typename ToHipType::MappedType HipTIn; - typedef typename ToHipType::MappedType HipTOut; - - // Get Input tensor count. - const auto total_tensor_count = ctx->InputCount(); - // We only have one tensor per group so - // grouped_tensor_pointers[i] always contains only one element. - std::vector> grouped_tensor_pointers(total_tensor_count); - std::vector tensor_sizes(total_tensor_count); - - for (int i = 0; i < total_tensor_count; ++i) { - const Tensor* input = ctx->Input(i); - const auto size = input->Shape().Size(); - ORT_ENFORCE(size <= std::numeric_limits::max(), "Number of reduced elements (", - size, ") exceeds the max allowed value (", std::numeric_limits::max(), ")."); - grouped_tensor_pointers[i] = {const_cast(input->Data())}; - tensor_sizes[i] = static_cast(size); - } - - // Allocate output tensor. - Tensor* output = ctx->Output(0, {}); - HipTOut* p_output = reinterpret_cast(output->template MutableData()); - HIP_RETURN_IF_ERROR(hipMemsetAsync(p_output, 0, sizeof(HipTOut), Stream(ctx))); - - // const bool deterministic = ctx->GetUseDeterministicCompute(); - bool deterministic = true; - - if (!deterministic) { - typedef MultiTensorReduceL2 TFunctor; - TFunctor functor; - - // Check if all values are finite and write true to deviceOutput. - // Otherwise, false will be written. - launch_multi_tensor_functor<1, TFunctor>(Stream(ctx), - 2048 * 32, tensor_sizes, grouped_tensor_pointers, functor, p_output); - - // *p_output is the squared sum of all elements. - // Let's take a sqrt to get the actual L2-norm. - ScalarSqrt(Stream(ctx), p_output, p_output); - } else { - // alternate path only for deterministic compute .. - typedef AccumulationType_t HipTAcc; - - // find reduction buffer size needed by 'reduce_square_sum' for each tensor - size_t reduction_buffer_size = 0; - for (int i = 0; i < total_tensor_count; ++i) { - reduction_buffer_size = - std::max(reduction_buffer_size, compute_reduction_buffer_size(tensor_sizes[i])); - } - - // enlarge reduction buffer size for 'reduce_sum' over tensor square norms - reduction_buffer_size = - std::max(reduction_buffer_size, compute_reduction_buffer_size(total_tensor_count)); - - // create GPU scratch space and zero target for each tensor square norm - auto reduction_buffer = GetScratchBuffer(reduction_buffer_size, ctx->GetComputeStream()); - - // buffer for final output and square norms of each tensor - auto results_buffer = GetScratchBuffer(1 + total_tensor_count, ctx->GetComputeStream()); - - HIP_RETURN_IF_ERROR(hipMemsetAsync(results_buffer.get(), 0, sizeof(HipTAcc) * (1 + total_tensor_count), Stream(ctx))); - - HipTAcc* p_global_sqnorm = results_buffer.get(); - HipTAcc* p_tensor_sqnorm = p_global_sqnorm + 1; - - // perform reduction l2norm = sqrt[sum(tensor[i][j]**2)] for i,j over all tensor elements - for (int i = 0; i < total_tensor_count; ++i) { - HipTIn* p_tensor_i = reinterpret_cast(grouped_tensor_pointers[i][0]); - ORT_RETURN_IF_ERROR(reduce_square_sum( - Stream(ctx), p_tensor_i, p_tensor_sqnorm + i, tensor_sizes[i], reduction_buffer.get(), reduction_buffer_size)); - } - ORT_RETURN_IF_ERROR(reduce_sum( - Stream(ctx), p_tensor_sqnorm, p_global_sqnorm, total_tensor_count, reduction_buffer.get(), reduction_buffer_size)); - ScalarSqrt(Stream(ctx), p_global_sqnorm, p_output); - } - - return Status::OK(); -} - -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, float, float) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, MLFloat16, float) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, float, MLFloat16) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, MLFloat16, MLFloat16) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, BFloat16, float) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, float, BFloat16) -REGISTER_REDUCE_ALL_KERNEL_TYPED(ReduceAllL2, BFloat16, BFloat16) - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc b/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc deleted file mode 100644 index 23811744885e0..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/reduction/reduction_ops.cc +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "orttraining/training_ops/rocm/reduction/reduction_ops.h" -#include "core/providers/common.h" -#include "core/providers/rocm/miopen_common.h" -#include "core/providers/rocm/math/unary_elementwise_ops_impl.h" -#include "core/providers/rocm/math/binary_elementwise_ops_impl.h" -#include "core/providers/rocm/math/binary_elementwise_ops.h" -#include "core/providers/cpu/tensor/utils.h" - -using namespace onnxruntime::common; -namespace onnxruntime { -namespace rocm { - -#define REGISTER_MS_KERNEL_TYPED(name, T) \ - ONNX_OPERATOR_TYPED_KERNEL_EX( \ - name, \ - kMSDomain, \ - 1, \ - T, \ - kRocmExecutionProvider, \ - (*KernelDefBuilder::Create()) \ - .InputMemoryType(OrtMemTypeCPUInput, 1) \ - .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ - name); - -REGISTER_MS_KERNEL_TYPED(ReduceSumTraining, MLFloat16) -REGISTER_MS_KERNEL_TYPED(ReduceSumTraining, float) -// REGISTER_MS_KERNEL_TYPED(ReduceSumTraining, double) -REGISTER_MS_KERNEL_TYPED(ReduceSumTraining, int32_t) - -template -template -Status ReduceKernel::ComputeImplEx(OpKernelContext* ctx, miopenReduceTensorOp_t miopen_reduce_op) const { - const Tensor* X = ctx->Input(0); - - // override the attribute value with the input value for reduction_axes - const Tensor* axes_tensor = ctx->Input(1); - ORT_ENFORCE(axes_tensor != nullptr, "Axes input is null"); - ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); - auto nDims = static_cast(axes_tensor->Shape()[0]); - const auto* data = axes_tensor->template Data(); - std::vector axes(data, data + nDims); - - // empty axes and no-op - if (axes.empty() && noop_with_empty_axes_) { - auto* Y = ctx->Output(0, X->Shape()); - HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), hipMemcpyDeviceToDevice, Stream(ctx))); - return Status::OK(); - } - - PrepareReduceMetadata prepare_reduce_metadata; - ORT_RETURN_IF_ERROR(PrepareForReduce(X, - keepdims_, - axes, - prepare_reduce_metadata)); - Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); - const bool fast_reduction = fast_reduction_ && !ctx->GetUseDeterministicCompute(); - - return ReduceComputeCore(Info().GetAllocator(OrtMemType::OrtMemTypeDefault), *X, prepare_reduce_metadata, *Y, miopen_reduce_op, axes, - calculate_log_, calculate_sqt_, log_sum_exp_, fast_reduction, ctx->GetComputeStream()); -} - -template <> -template <> -Status ReduceKernel::ComputeImplEx(OpKernelContext* ctx, miopenReduceTensorOp_t miopen_reduce_op) const { - typedef typename ToHipType::MappedType HipT; - - const Tensor* X = ctx->Input(0); - - // override the attribute value with the input value for reduction_axes - const Tensor* axes_tensor = ctx->Input(1); - ORT_ENFORCE(axes_tensor->Shape().NumDimensions() == 1, "An axes tensor must be a vector tensor."); - auto nDims = static_cast(axes_tensor->Shape()[0]); - const auto* data = axes_tensor->template Data(); - std::vector axes(data, data + nDims); - - // empty axes and no-op - if (axes.empty() && noop_with_empty_axes_) { - auto* Y = ctx->Output(0, X->Shape()); - HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), X->SizeInBytes(), hipMemcpyDeviceToDevice, Stream(ctx))); - return Status::OK(); - } - - PrepareReduceMetadata prepare_reduce_metadata; - - ORT_RETURN_IF_ERROR(PrepareForReduce(X, - keepdims_, - axes, - prepare_reduce_metadata)); - - Tensor* Y = ctx->Output(0, prepare_reduce_metadata.squeezed_output_dims); - - int64_t input_count = prepare_reduce_metadata.input_count; - int64_t output_count = prepare_reduce_metadata.output_count; - auto& input_dims_miopen = prepare_reduce_metadata.input_dims_miopen; - auto& output_dims_miopen = prepare_reduce_metadata.output_dims_miopen; - - // special case when there is a dim value of 0 in the shape. - if (input_count == 0) { - assert(Y->Shape().Size() == 0); - return Status::OK(); - } - - // miopenReduceTensor for ReduceSum has issue if input and output has same size, we just need to copy the data for this case - if (input_count == output_count) { - if (Y->template MutableData() != X->template Data()) { - HIP_RETURN_IF_ERROR(hipMemcpyAsync(Y->template MutableData(), X->template Data(), input_count * sizeof(int32_t), hipMemcpyDeviceToDevice, Stream(ctx))); - } - return Status::OK(); - } - - // This reduction keep adding values to this buffer. If a non-zero value, say 1000, is here, the sum will start with 1000. - // Therefore zeroing out the memory is required - HIP_RETURN_IF_ERROR(hipMemsetAsync(Y->MutableDataRaw(), 0, Y->SizeInBytes(), Stream(ctx))); - - size_t indices_bytes = 0; - size_t workspace_bytes = 0; - MiopenTensor input_tensor; - MiopenTensor output_tensor; - MiopenReduceDescriptor reduce_desc; - - miopenDataType_t miopen_type_X = miopenFloat; - IAllocatorUniquePtr temp_X = GetScratchBuffer(input_count, ctx->GetComputeStream()); - Impl_Cast(Stream(ctx), reinterpret_cast(X->template Data()), temp_X.get(), X->Shape().Size()); - - ORT_RETURN_IF_ERROR(reduce_desc.Set(miopen_reduce_op, miopen_type_X, MIOPEN_REDUCE_TENSOR_FLATTENED_INDICES)); - ORT_RETURN_IF_ERROR(input_tensor.Set(input_dims_miopen, miopen_type_X)); - ORT_RETURN_IF_ERROR(output_tensor.Set(output_dims_miopen, miopen_type_X)); - MIOPEN_RETURN_IF_ERROR(miopenGetReductionIndicesSize(GetMiopenHandle(ctx), reduce_desc, input_tensor, output_tensor, &indices_bytes)); - MIOPEN_RETURN_IF_ERROR(miopenGetReductionWorkspaceSize(GetMiopenHandle(ctx), reduce_desc, input_tensor, output_tensor, &workspace_bytes)); - IAllocatorUniquePtr indices_rocm = GetScratchBuffer(indices_bytes, ctx->GetComputeStream()); - IAllocatorUniquePtr workspace_rocm = GetScratchBuffer(workspace_bytes, ctx->GetComputeStream()); - - const auto one = Consts::One; - const auto zero = Consts::Zero; - auto temp_Y = GetScratchBuffer(output_count, ctx->GetComputeStream()); - MIOPEN_RETURN_IF_ERROR(miopenReduceTensor(GetMiopenHandle(ctx), - reduce_desc, - indices_rocm.get(), - indices_bytes, - workspace_rocm.get(), - workspace_bytes, - &one, - input_tensor, - temp_X.get(), - &zero, - output_tensor, - temp_Y.get())); - - Impl_Cast(Stream(ctx), temp_Y.get(), Y->template MutableData(), output_count); - - return Status::OK(); -} - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc b/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc deleted file mode 100644 index c570f727f2a92..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc +++ /dev/null @@ -1,437 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/shared_library/provider_api.h" -#include "core/providers/rocm/rocm_fwd.h" -#include "core/providers/rocm/rocm_pch.h" - -using namespace onnxruntime::common; - -namespace onnxruntime { -namespace rocm { - -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, View); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, Group); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, PassThrough); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SGDOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ReduceSumTraining); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, ReduceSumTraining); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, int32_t, ReduceSumTraining); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ReduceSumTraining); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SplitTraining); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, ConcatTraining); - -// Adam -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_float_float_float_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_float_MLFloat16_float_float_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_MLFloat16_float_float_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_float_MLFloat16_MLFloat16_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_float_MLFloat16_float_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_float_MLFloat16_MLFloat16_MLFloat16_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_float_MLFloat16_MLFloat16_float_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_MLFloat16_MLFloat16_MLFloat16_MLFloat16, AdamOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float_MLFloat16_MLFloat16_float_MLFloat16, AdamOptimizer); -// Lamb -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float_float_float_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_MLFloat16_float_MLFloat16_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_MLFloat16_float_float_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double_double_double_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16_MLFloat16_MLFloat16_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16_MLFloat16_float_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16_float_MLFloat16_MLFloat16, LambOptimizer); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16_float_float_MLFloat16, LambOptimizer); -// Gradient accumulator -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float, InPlaceAccumulator); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_MLFloat16, InPlaceAccumulator); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16, InPlaceAccumulator); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float, InPlaceAccumulator); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ZeroGradient); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ZeroGradient); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, SoftmaxCrossEntropy); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, SoftmaxCrossEntropyGrad); -// class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 9, float, int32_t, SparseSoftmaxCrossEntropy); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 9, float, int64_t, SparseSoftmaxCrossEntropy); -// class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 9, float, int32_t, SparseSoftmaxCrossEntropyGrad); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 9, float, int64_t, SparseSoftmaxCrossEntropyGrad); -class ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 12, 12, MLFloat16, int64_t, SoftmaxCrossEntropyLoss); -class ONNX_OPERATOR_VERSIONED_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 12, 12, float, int64_t, SoftmaxCrossEntropyLoss); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 13, MLFloat16, int64_t, SoftmaxCrossEntropyLoss); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 13, float, int64_t, SoftmaxCrossEntropyLoss); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kOnnxDomain, 13, BFloat16, int64_t, SoftmaxCrossEntropyLoss); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, int64_t, SoftmaxCrossEntropyLossGrad); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, int64_t, SoftmaxCrossEntropyLossGrad); -class ONNX_OPERATOR_TWO_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16, int64_t, SoftmaxCrossEntropyLossGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_float, SoftmaxCrossEntropyLossInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float, SoftmaxCrossEntropyLossInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_MLFloat16, SoftmaxCrossEntropyLossInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16_int64_t_BFloat16, SoftmaxCrossEntropyLossInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_MLFloat16, SoftmaxCrossEntropyLossInternalGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_int64_t_float, SoftmaxCrossEntropyLossInternalGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_int64_t_MLFloat16, SoftmaxCrossEntropyLossInternalGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16_int64_t_BFloat16, SoftmaxCrossEntropyLossInternalGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SoftmaxGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, LogSoftmaxGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SoftmaxGrad_13); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, LogSoftmaxGrad_13); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float, BatchNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double, BatchNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16_MLFloat16, BatchNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16_float, BatchNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_float, BatchNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ConvGrad); -// class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, ConvGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ConvGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, GatherGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, DropoutGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BitmaskDropoutGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasSoftmaxDropout); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SoftmaxDropoutGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, int64_t, GatherNDGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, DivGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, DivGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, DivGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, GeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, GeluGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, FastGeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, FastGeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, FastGeluGrad); - -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasGeluGrad_dX); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasFastGeluGrad_dX); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ReluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, ReluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ReluGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, SigmoidGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, SigmoidGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, SigmoidGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, QuickGeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, QuickGeluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, QuickGeluGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, TanhGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, TanhGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, TanhGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, LeakyReluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, LeakyReluGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, LeakyReluGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, IsFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, IsFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, IsFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, bool, All); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, IsAllFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, IsAllFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, IsAllFinite); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, MixedPrecisionScale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, MixedPrecisionScale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_MLFloat16, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float, LayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double, LayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16, LayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_MLFloat16, LayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_float, LayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float, SimplifiedLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double, SimplifiedLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16, SimplifiedLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_MLFloat16, SimplifiedLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_float, SimplifiedLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float, InvertibleLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double, InvertibleLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_MLFloat16, InvertibleLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_MLFloat16, InvertibleLayerNormalizationGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_float, InvertibleLayerNormalizationGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, SliceGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, GatherElementsGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, Scale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, Scale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, Scale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16, Scale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistBinarizeEncoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, GistBinarizeEncoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, GistBinarizeEncoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistBinarizeDecoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, GistBinarizeDecoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, GistBinarizeDecoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, bool, GistPack1Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack1Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, bool, GistPack1Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack1Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack8Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, GistPack8Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack8Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, GistPack8Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack16Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPack16Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPackMsfp15Encoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, GistPackMsfp15Decoder); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_float_float, BatchNormInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double_double_double, BatchNormInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16_MLFloat16, BatchNormInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_MLFloat16_float, BatchNormInternal); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16_float_float, BatchNormInternal); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16, MixedPrecisionScale); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16_float_BFloat16, LayerNormalizationGrad); - -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16_float, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float_BFloat16, ReduceAllL2); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BFloat16_BFloat16, ReduceAllL2); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, PadAndUnflatten); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, FlattenAndUnpad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ResizeGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ResizeGrad); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, ResizeGrad); - -#if defined(ORT_USE_NCCL) || defined(USE_MPI) -// P2P communication operators. -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, Send); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, Recv); -#endif - -#ifdef USE_MPI -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, AdasumAllReduce); -#endif - -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, RecordEvent); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, WaitEvent); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, YieldOp); - -#ifdef ENABLE_TRAINING_TORCH_INTEROP -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, PythonOp); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, PythonOpGrad); -#endif - -#ifdef ORT_USE_NCCL -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, NcclAllReduce); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, NcclAllGather); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, NcclReduceScatter); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MegatronF); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MegatronG); -#endif - -Status RegisterRocmTrainingKernels(KernelRegistry& kernel_registry) { - static const BuildKernelCreateInfoFn function_table[] = { - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // Adam - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - // Lamb - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - // BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - -// P2P communication operators. -#if defined(ORT_USE_NCCL) || defined(USE_MPI) - BuildKernelCreateInfo, - BuildKernelCreateInfo, -#endif - -#ifdef USE_MPI - // BuildKernelCreateInfo, -#endif - - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - -#ifdef ENABLE_TRAINING_TORCH_INTEROP - BuildKernelCreateInfo, - BuildKernelCreateInfo, -#endif - -#ifdef ORT_USE_NCCL - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, -#endif - }; - - for (auto& function_table_entry : function_table) { - ORT_RETURN_IF_ERROR(kernel_registry.Register(function_table_entry())); - } - - return Status::OK(); -} - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.h b/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.h deleted file mode 100644 index 697975b7f3409..0000000000000 --- a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#pragma once - -namespace onnxruntime { -namespace rocm { - -Status RegisterRocmTrainingKernels(KernelRegistry& kernel_registry); - -} // namespace rocm -} // namespace onnxruntime diff --git a/orttraining/tools/amdgpu/Dockerfile.rocm4.3.1.pytorch b/orttraining/tools/amdgpu/Dockerfile.rocm4.3.1.pytorch deleted file mode 100644 index 29b8812c979e4..0000000000000 --- a/orttraining/tools/amdgpu/Dockerfile.rocm4.3.1.pytorch +++ /dev/null @@ -1,170 +0,0 @@ -# docker build --network=host --file Dockerfile.rocm4.3.1.pytorch --tag ort:rocm4.3.1-pytorch . - -FROM rocm/pytorch:rocm4.3.1_ubuntu18.04_py3.6_pytorch_1.9.0 - -RUN apt-get -y install gpg-agent -RUN wget -q -O - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - -RUN echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/4.3.1 xenial main' | tee /etc/apt/sources.list.d/rocm.list - -RUN apt-get -y update -RUN apt-get -y install apt-utils -RUN apt-get -y install build-essential autotools-dev \ - make git curl vim wget rsync jq openssh-server openssh-client sudo \ - iputils-ping net-tools ethtool libcap2 \ - automake autoconf libtool flex doxygen \ - perl lsb-release iproute2 pciutils graphviz \ - bc tar git bash pbzip2 pv bzip2 unzip cabextract \ - g++ gcc \ - && apt-get autoremove - -# sh -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# Labels for the docker -LABEL description="This docker sets up the environment to run ORT Training with AMD GPU" - -# CMake -ENV CMAKE_VERSION=3.18.2 -RUN cd /usr/local && \ - wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - -ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:${PATH} - -ENV WORKSPACE_DIR=/workspace -RUN mkdir -p $WORKSPACE_DIR -WORKDIR $WORKSPACE_DIR - -ENV OLD_PATH=${PATH} -ENV PATH=/usr/bin:${PATH} -# Infiniband setup, openmpi installed under /usr/mpi/gcc/openmpi-4.0.4rc3 doesn't support multi-thread -ENV MOFED_VERSION=5.1-0.6.6.0 -ENV MOFED_OS=ubuntu18.04 -ENV MOFED_FILENAME=MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 -RUN curl -fSsL https://www.mellanox.com/downloads/ofed/MLNX_OFED-${MOFED_VERSION}/${MOFED_FILENAME}.tgz | tar -zxpf - -RUN cd MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 && \ - ./mlnxofedinstall --force --user-space-only --without-fw-update --hpc && \ - cd .. && \ - rm -r MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 - -ENV PATH=${OLD_PATH} -ENV unset=OLD_PATH - -# python env -RUN pip3 install --upgrade setuptools -ARG NUMPY_VERSION=1.18.5 -ARG ONNX_VERSION=1.10.2 -RUN pip3 install --no-cache-dir wheel tqdm boto3 requests six ipdb h5py html2text nltk progressbar pyyaml \ - git+https://github.com/NVIDIA/dllogger \ - numpy==${NUMPY_VERSION} \ - onnx=="${ONNX_VERSION}" - -ENV GITHUB_DIR=$WORKSPACE_DIR/github -RUN mkdir -p $GITHUB_DIR - -# UCX -WORKDIR $GITHUB_DIR -RUN apt-get -y update && apt-get -y --no-install-recommends install libnuma-dev -ARG UCX_VERSION=1.9.0-rc3 -ENV UCX_DIR=$WORKSPACE_DIR/ucx-$UCX_VERSION -RUN git clone https://github.com/openucx/ucx.git \ - && cd ucx \ - && git checkout v$UCX_VERSION \ - && ./autogen.sh \ - && mkdir build \ - && cd build \ - && ../contrib/configure-opt --prefix=$UCX_DIR --without-rocm --without-knem --without-cuda \ - && make -j"$(nproc)" \ - && make install \ - && cd .. \ - && rm -rf build - -# OpenMPI -# note: require --enable-orterun-prefix-by-default for Azure machine learning compute -# note: disable verbs as we use ucx middleware and don't want btl openib warnings -WORKDIR $GITHUB_DIR -ARG OPENMPI_BASEVERSION=4.0 -ARG OPENMPI_VERSION=${OPENMPI_BASEVERSION}.5 -ENV OPENMPI_DIR=$WORKSPACE_DIR/openmpi-${OPENMPI_VERSION} -RUN git clone --recursive https://github.com/open-mpi/ompi.git \ - && cd ompi \ - && git checkout v$OPENMPI_VERSION \ - && ./autogen.pl \ - && mkdir build \ - && cd build \ - && ../configure --prefix=$OPENMPI_DIR --with-ucx=$UCX_DIR --without-verbs \ - --enable-mpirun-prefix-by-default --enable-orterun-prefix-by-default \ - --enable-mca-no-build=btl-uct --disable-mpi-fortran \ - && make -j"$(nproc)" \ - && make install \ - && cd .. \ - && rm -rf build \ - && ldconfig \ - && test -f ${OPENMPI_DIR}/bin/mpic++ - -ENV PATH=$OPENMPI_DIR/bin:${PATH} -ENV LD_LIBRARY_PATH=$OPENMPI_DIR/lib:${LD_LIBRARY_PATH} - -# Create a wrapper for OpenMPI to allow running as root by default -RUN mv $OPENMPI_DIR/bin/mpirun $OPENMPI_DIR/bin/mpirun.real && \ - echo '#!/bin/bash' > $OPENMPI_DIR/bin/mpirun && \ - echo 'mpirun.real --allow-run-as-root "$@"' >> $OPENMPI_DIR/bin/mpirun && \ - chmod a+x $OPENMPI_DIR/bin/mpirun - -# install mpi4py (be sure to link existing /opt/openmpi-xxx) -RUN CC=mpicc MPICC=mpicc pip install mpi4py --no-binary mpi4py - -ARG CACHE_DATA=2021-10-25 - -# ONNX Runtime -WORKDIR $GITHUB_DIR -ENV ORT_DIR=$GITHUB_DIR/onnxruntime -RUN git clone -b wezhan/tnlrv4 --recursive https://github.com/microsoft/onnxruntime.git \ - && cd onnxruntime \ - && python3 tools/ci_build/build.py \ - --cmake_extra_defines ONNXRUNTIME_VERSION=`cat ./VERSION_NUMBER` \ - --build_dir build \ - --config Release \ - --parallel \ - --skip_tests \ - --build_wheel \ - --use_rocm --rocm_version=4.3.1 --rocm_home /opt/rocm \ - --mpi_home $OPENMPI_DIR \ - --nccl_home /opt/rocm \ - --enable_training \ - && test -f $ORT_DIR/build/Release/onnxruntime_training_bert \ - && pip install $ORT_DIR/build/Release/dist/*.whl \ - && ldconfig - -RUN pip3 install --no-cache-dir GPUtil azureml azureml-core datasets tokenizers ninja cerberus sympy sacremoses sacrebleu - -RUN pip install transformers==2.10.0 scikit-learn tensorboardX -RUN pip install --pre torch-ort -f https://download.onnxruntime.ai/torch_ort_nightly.html -RUN python -m torch_ort.configure - -# Enable ssh access without password needed -RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config -RUN sed -i 's/#StrictModes yes/StrictModes no/g' /etc/ssh/sshd_config -RUN sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config -RUN sed -i 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/g' /etc/ssh/sshd_config - -# Start or Restart sshd service -ENTRYPOINT service ssh restart && /bin/bash - -# Add model and scripts -ADD script ${WORKSPACE_DIR}/script -RUN chmod a+x ${WORKSPACE_DIR}/script/run_bert.sh - -# add locale en_US.UTF-8 -RUN apt-get install -y locales -RUN locale-gen en_US.UTF-8 - -# Workaround an issue in AMD compiler which generates poor GPU ISA -# when the type of kernel parameter is a structure and “pass-by-value” is used -# ENV HSA_NO_SCRATCH_RECLAIM=1 - -# Distributed training related environment variables -ENV HSA_FORCE_FINE_GRAIN_PCIE=1 -# ENV NCCL_DEBUG=INFO -# ENV RCCL_ALLTOALL_KERNEL_DISABLE=1 -# ENV NCCL_DEBUG_SUBSYS=INIT,COLL - -WORKDIR ${WORKSPACE_DIR}/script diff --git a/orttraining/tools/amdgpu/script/rocprof.py b/orttraining/tools/amdgpu/script/rocprof.py deleted file mode 100644 index 21dd8501f3f1d..0000000000000 --- a/orttraining/tools/amdgpu/script/rocprof.py +++ /dev/null @@ -1,77 +0,0 @@ -import argparse -import csv -import os # noqa: F401 - -import numpy as np # noqa: F401 - -parser = argparse.ArgumentParser() -parser.add_argument("--input", type=str) -args = parser.parse_args() - - -def get_gpu_lines(path): - lines = [] - with open(path, newline="") as f: - reader = csv.reader(f, delimiter=",") - for row in reader: - if row[2].find("TotalDurationNs") < 0: - lines.append(row) - return lines - - -activities = [ - ("nccl", lambda x: x.find("nccl") >= 0), - ("gemm", lambda x: x.find("Cijk_") >= 0), - ("memcpy", lambda x: x.find("CUDA mem") >= 0), - ("adam", lambda x: x.lower().find("adam") >= 0), - ("lamb", lambda x: x.lower().find("lamb") >= 0 or x.lower().find("multi_tensor_apply") >= 0), - ("dropout", lambda x: x.lower().find("dropout") >= 0 or x.find("curand") >= 0), - ("layernorm", lambda x: x.find("LayerNorm") >= 0 or x.find("cuCompute") >= 0), - ("reduce", lambda x: x.find("reduce") >= 0), - ("softmax", lambda x: x.lower().find("softmax") >= 0), - ("transpose", lambda x: x.lower().find("transpose") >= 0), - ("element-wise", lambda x: x.lower().find("elementwise") >= 0 or x.find("DivGrad") >= 0), - ("jit", lambda x: x.startswith("kernel_")), - ("misc", lambda x: True), -] - - -def group_gpu_activity(lines): - groups = {name: [] for name, _ in activities} - for line in lines: - for name, check in activities: - if check(line[0]): - groups[name].append(line) - break - return groups - - -def get_seconds(time): - return float(time.replace("us", "")) / (1000.0 * 1000.0 * 1000.0) - - -def gpu_percent_time(activities): - return sum([float(a[4].replace("%", "")) for a in activities]) - - -def gpu_absolute_time(activities): - return sum([get_seconds(a[2]) for a in activities]) - - -def gpu_kernel_calls(activities): - return sum([int(a[1]) for a in activities]) - - -lines = get_gpu_lines(args.input) -groups = group_gpu_activity(lines) - -for name in groups: - activities = groups[name] - print( - f"{name}: N={len(activities)}, calls={gpu_kernel_calls(activities)}, absolute={gpu_absolute_time(activities):.3f}s, percent={gpu_percent_time(activities):.2f}%" - ) - -total = [item for name in groups for item in groups[name]] -print( - f"Total: N={len(total)}, calls={gpu_kernel_calls(total)}, absolute={gpu_absolute_time(total):.3f}s, percent={gpu_percent_time(total):.2f}%" -) diff --git a/orttraining/tools/amdgpu/script/rpl_rc.xml b/orttraining/tools/amdgpu/script/rpl_rc.xml deleted file mode 100644 index 3ca51072b6c98..0000000000000 --- a/orttraining/tools/amdgpu/script/rpl_rc.xml +++ /dev/null @@ -1,10 +0,0 @@ - - diff --git a/orttraining/tools/amdgpu/script/run_bert.sh b/orttraining/tools/amdgpu/script/run_bert.sh deleted file mode 100644 index 950dcaf89ff61..0000000000000 --- a/orttraining/tools/amdgpu/script/run_bert.sh +++ /dev/null @@ -1,84 +0,0 @@ -if [ "$#" -ne 12 ]; then - echo "Usage: $0 ngpu batch_size seq_len num_train_steps optimizer model_size training_mode[fp32|fp16] display_loss_steps gradient_accumulation_steps loss_scale gpu_name profile" - exit 1 -fi - -ngpu=${1:-1} -batch_size=${2:-64} -seq_len=${3:-128} - -if [ ${seq_len} == 128 ]; then - max_predictions_per_seq=20 -elif [ ${seq_len} == 512 ]; then - max_predictions_per_seq=80 -else - echo "seq_len is not 128 or 512" - exit 1 -fi - -num_train_steps=${4:-400} -optimizer=${5:-"adam"} -model_size=${6:-"large"} -training_mode=${7:-"fp32"} -display_loss_steps=${8:-1} -grad_acc=${9:-1} -loss_scale=${10:-1024} -gpu_name=${11:-"mi100"} -profile=${12:-0} - -lr=5e-5 -warmup_ratio=0.2843 -warmup_mode=Poly -effective_batch_size=$((ngpu * batch_size * grad_acc)) -time_now=$(date +%m%d%H%M) - -HOME_DIR=/workspace -ORT_DIR=${HOME_DIR}/github/onnxruntime -commit=$(git -C ${ORT_DIR} rev-parse HEAD | cut -c1-8) - -if [ ${model_size} == "large" ]; then - model_dir=${HOME_DIR}/model/bert-large-uncased_L_24_H_1024_A_16_V_30528_S_512_Dp_0.1_optimized_layer_norm_opset12 -elif [ ${model_size} == "base" ]; then - model_dir=${HOME_DIR}/model/bert-base-uncased_L_12_H_768_A_12_V_30528_S_512_Dp_0.1_optimized_layer_norm_opset12 -elif [ ${model_size} == "tiny" ]; then - model_dir=${HOME_DIR}/model/bert-tiny-uncased_L_3_H_128_A_2_V_30528_S_512_Dp_0.1_optimized_layer_norm_opset12 -else - echo "model_size is not large, base or tiny" - exit 1 -fi - -data_dir=/data/wezhan/bert/${seq_len}/train -training_bert_dir=${ORT_DIR}/build/RelWithDebInfo - -log_dir=${HOME_DIR}/logs/bert_${model_size}/$(date +%m%d) -if [ ! -d ${log_dir} ]; then - mkdir -p ${log_dir} -fi - -run_name=bert_${model_size}_${commit}_g${ngpu}_bs${batch_size}_sl${seq_len}_steps${num_train_steps}_${optimizer}_${training_mode}_acc${grad_acc}_efbs${effective_batch_size}_${time_now}_${gpu_name} - -if [ ! -d ${log_dir}/${run_name} ]; then - mkdir -p ${log_dir}/${run_name} -fi - -if [ ${ngpu} != 1 ]; then - mpi_cmd="${OPENMPI_DIR}/bin/mpirun --allow-run-as-root -n ${ngpu} -x NCCL_DEBUG=INFO -x NCCL_DEBUG_SUBSYS=INIT,COLL -x NCCL_MIN_NCHANNELS=4" -fi - -if [ ${training_mode} == "fp16" ]; then - fp16_commands="--use_mixed_precision --allreduce_in_fp16 --loss_scale ${loss_scale}" -fi - -if [ ${profile} == 1 ]; then - if [ ${gpu_name} == "mi100" ]; then - profile_commands="/opt/rocm/bin/rocprof --obj-tracking on --stats" - elif [ ${gpu_name} == "v100" ]; then - profile_commands="nvprof --print-gpu-summary --log-file ${log_dir}/${run_name}-trace.log" - fi -fi - -nohup ${profile_commands} ${mpi_cmd} ${training_bert_dir}/onnxruntime_training_bert --model_name ${model_dir} --train_data_dir ${data_dir} --test_data_dir ${data_dir} --train_batch_size ${batch_size} --mode train --num_train_steps ${num_train_steps} --optimizer ${optimizer} --learning_rate ${lr} --warmup_ratio ${warmup_ratio} --warmup_mode ${warmup_mode} --gradient_accumulation_steps ${grad_acc} --max_seq_length ${seq_len} --max_predictions_per_seq=${max_predictions_per_seq} --use_nccl --lambda 0 ${fp16_commands} --display_loss_steps ${display_loss_steps} --log_dir ${log_dir}/${run_name} > ${log_dir}/${run_name}.log 2>&1 & - -tail -f ${log_dir}/${run_name}.log - -exit 0 diff --git a/setup.py b/setup.py index f6a697b1bb2b9..dd495da56c4c3 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,6 @@ def parse_arg_remove_string(argv, arg_name_equal): cuda_version = None cuda_major_version = None -rocm_version = None is_migraphx = False is_openvino = False is_qnn = False @@ -244,7 +243,7 @@ def run(self): "libnvrtc-builtins.so.13", ] - rocm_dependencies = [ + migraphx_dependencies = [ "libamd_comgr.so.2", "libamdhip64.so.5", "libamdhip64.so.6", @@ -300,7 +299,7 @@ def run(self): file = glob(path.join(self.dist_dir, "*linux*.whl"))[0] logger.info("repairing %s for manylinux1", file) auditwheel_cmd = ["auditwheel", "-v", "repair", "-w", self.dist_dir, file] - for i in cuda_dependencies + rocm_dependencies + tensorrt_dependencies + cann_dependencies: + for i in cuda_dependencies + migraphx_dependencies + tensorrt_dependencies + cann_dependencies: auditwheel_cmd += ["--exclude", i] logger.info("Running %s", " ".join([shlex.quote(arg) for arg in auditwheel_cmd])) try: @@ -322,7 +321,7 @@ def finalize_options(self): return ret -providers_cuda_or_rocm = "onnxruntime_providers_cuda" +providers_cuda = "onnxruntime_providers_cuda" providers_tensorrt_or_migraphx = "onnxruntime_providers_" + ("migraphx" if is_migraphx else "tensorrt") providers_nv_tensorrt_rtx = "onnxruntime_providers_nv_tensorrt_rtx" providers_openvino = "onnxruntime_providers_openvino" @@ -330,14 +329,14 @@ def finalize_options(self): providers_qnn = "onnxruntime_providers_qnn" if platform.system() == "Linux": - providers_cuda_or_rocm = "lib" + providers_cuda_or_rocm + ".so" + providers_cuda = "lib" + providers_cuda + ".so" providers_tensorrt_or_migraphx = "lib" + providers_tensorrt_or_migraphx + ".so" providers_nv_tensorrt_rtx = "lib" + providers_nv_tensorrt_rtx + ".so" providers_openvino = "lib" + providers_openvino + ".so" providers_cann = "lib" + providers_cann + ".so" providers_qnn = "lib" + providers_qnn + ".so" elif platform.system() == "Windows": - providers_cuda_or_rocm = providers_cuda_or_rocm + ".dll" + providers_cuda = providers_cuda + ".dll" providers_tensorrt_or_migraphx = providers_tensorrt_or_migraphx + ".dll" providers_nv_tensorrt_rtx = providers_nv_tensorrt_rtx + ".dll" providers_openvino = providers_openvino + ".dll" @@ -359,7 +358,7 @@ def finalize_options(self): "libonnxruntime.so*", ] dl_libs = ["libonnxruntime_providers_shared.so"] - dl_libs.append(providers_cuda_or_rocm) + dl_libs.append(providers_cuda) dl_libs.append(providers_tensorrt_or_migraphx) dl_libs.append(providers_cann) dl_libs.append(providers_qnn) @@ -369,7 +368,7 @@ def finalize_options(self): libs.extend(["libonnxruntime_providers_dnnl.so"]) libs.extend(["libonnxruntime_providers_openvino.so"]) libs.extend(["libonnxruntime_providers_vitisai.so"]) - libs.append(providers_cuda_or_rocm) + libs.append(providers_cuda) libs.append(providers_nv_tensorrt_rtx) libs.append(providers_tensorrt_or_migraphx) libs.append(providers_cann) @@ -410,7 +409,7 @@ def finalize_options(self): "dnnl.dll", "mklml.dll", "libiomp5md.dll", - providers_cuda_or_rocm, + providers_cuda, providers_tensorrt_or_migraphx, providers_nv_tensorrt_rtx, providers_cann, @@ -680,21 +679,14 @@ def finalize_options(self): if cuda_version: # removing '.' to make Cuda version number in the same form as Pytorch. local_version = "+cu" + cuda_version.replace(".", "") - elif rocm_version: - # removing '.' to make Rocm version number in the same form as Pytorch. - local_version = "+rocm" + rocm_version.replace(".", "") else: # cpu version for documentation local_version = "+cpu" else: - if not (cuda_version or rocm_version): + if not cuda_version: # Training CPU package for ADO feeds is called onnxruntime-training-cpu package_name = "onnxruntime-training-cpu" - if rocm_version: - # Training ROCM package for ADO feeds is called onnxruntime-training-rocm - package_name = "onnxruntime-training-rocm" - if package_name == "onnxruntime-tvm": packages += ["onnxruntime.providers.tvm"] @@ -796,7 +788,7 @@ def reformat_run_count(count_str): install_requires.append(f"nvidia-cuda-runtime-cu{major}~={major}.0") -def save_build_and_package_info(package_name, version_number, cuda_version, rocm_version, qnn_version): +def save_build_and_package_info(package_name, version_number, cuda_version, qnn_version): sys.path.append(path.join(path.dirname(__file__), "onnxruntime", "python")) from onnxruntime_collect_build_info import find_cudart_versions # noqa: PLC0415 @@ -823,13 +815,11 @@ def save_build_and_package_info(package_name, version_number, cuda_version, rocm else "found multiple cudart libraries" ), ) - elif rocm_version: - f.write(f"rocm_version = '{rocm_version}'\n") elif qnn_version: f.write(f"qnn_version = '{qnn_version}'\n") -save_build_and_package_info(package_name, version_number, cuda_version, rocm_version, qnn_version) +save_build_and_package_info(package_name, version_number, cuda_version, qnn_version) extras_require = {} if package_name == "onnxruntime-gpu" and cuda_major_version: diff --git a/tools/ci_build/amd_hipify.py b/tools/ci_build/amd_hipify.py deleted file mode 100644 index 6a8154681ed97..0000000000000 --- a/tools/ci_build/amd_hipify.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import argparse -import os -import subprocess - - -def hipify(hipify_perl_path, src_file_path, dst_file_path): - dir_name = os.path.dirname(dst_file_path) - if not os.path.exists(dir_name): - os.makedirs(dir_name, exist_ok=True) - # Run hipify-perl first, capture output - s = subprocess.run([hipify_perl_path, src_file_path], stdout=subprocess.PIPE, text=True, check=False).stdout - - # Additional exact-match replacements. - # Order matters for all of the following replacements, reglardless of appearing in logical sections. - s = s.replace("kCudaExecutionProvider", "kRocmExecutionProvider") - s = s.replace("CUDAStreamType", "HIPStreamType") - s = s.replace("kCudaStreamDefault", "kHipStreamDefault") - s = s.replace("kCudaStreamCopyIn", "kHipStreamCopyIn") - s = s.replace("kCudaStreamCopyOut", "kHipStreamCopyOut") - s = s.replace("kTotalCudaStreams", "kTotalHipStreams") - - # in rocm 6.0, hipify-perl, the -roc option also maps __half -> rocblas_half which we don't want - s = s.replace("rocblas_half", "__half") - - # these should be "hip" but it's easier to just use rocm to avoid complicated file renaming - s = s.replace("CudaGraph", "RocmGraph") - s = s.replace("CUDAGraph", "ROCMGraph") - s = s.replace("cuda_graph", "rocm_graph") - s = s.replace("RegisterCudaContribKernels", "RegisterRocmContribKernels") - s = s.replace("cudaEvent", "hipEvent") - s = s.replace("CreateCudaAllocator", "CreateRocmAllocator") - s = s.replace("CudaErrString", "RocmErrString") - s = s.replace("CudaAsyncBuffer", "RocmAsyncBuffer") - s = s.replace("CudaKernel", "RocmKernel") - s = s.replace("CudaStream", "RocmStream") - s = s.replace("ToCudaType", "ToHipType") - s = s.replace("CudaT", "HipT") - s = s.replace("CUDA_LONG", "HIP_LONG") - s = s.replace("CUDA_RETURN_IF_ERROR", "HIP_RETURN_IF_ERROR") - s = s.replace("CUDA_KERNEL_ASSERT", "HIP_KERNEL_ASSERT") - s = s.replace("CUDA_CALL", "HIP_CALL") - s = s.replace("SliceCuda", "SliceRocm") - s = s.replace("thrust::cuda", "thrust::hip") - s = s.replace("CudaCall", "RocmCall") - s = s.replace("cuda", "rocm") - # s = s.replace('Cuda', 'Rocm') - s = s.replace("CUDA", "ROCM") - s = s.replace("GPU_WARP_SIZE = 32", "GPU_WARP_SIZE = 64") - s = s.replace("std::exp", "expf") - s = s.replace("std::log", "logf") - s = s.replace("WaitCudaNotificationOnDevice", "WaitRocmNotificationOnDevice") - s = s.replace("hipHostAlloc", "hipHostMalloc") - s = s.replace( - "#include ", - "#include \n#include ", - ) - s = s.replace( - '#include "cub/device/device_radix_sort.cuh"', - "#include \n#include ", - ) - s = s.replace( - "#include ", - "#include ", - ) - s = s.replace( - "#include ", "#include " - ) - s = s.replace( - "#include ", - "#include ", - ) - s = s.replace("#include ", "#include ") - s = s.replace( - "#include ", - "#include ", - ) - s = s.replace( - "#include ", - "#include ", - ) - s = s.replace("#include ", "#include ") - s = s.replace('#include "cub/util_allocator.cuh"', "#include ") - s = s.replace("#include ", "#include ") - s = s.replace('#include "cub/util_type.cuh"', "#include ") - s = s.replace("#include ", "#include ") - s = s.replace("#include ", "#include ") - s = s.replace("#include ", "") # Doesn't exist - s = s.replace("typedef half MappedType", "typedef __half MappedType") - - # CUBLAS -> HIPBLAS - s = s.replace("CUBLAS", "HIPBLAS") - s = s.replace("Cublas", "Hipblas") - s = s.replace("cublas", "hipblas") - # deprecated cublas symbol doesn't exist in hipblas, map to new symbol - s = s.replace("HIPBLAS_GEMM_DEFAULT_TENSOR_OP", "HIPBLAS_GEMM_DEFAULT") - - # Undefined ROCMRT constants -> std::numeric_limits - s = s.replace("ROCMRT_INF_F", "std::numeric_limits::infinity()") - - # compatible layer - s = s.replace("rocblas_gemm_strided_batched_ex", "_compat_rocblas_gemm_strided_batched_ex") - s = s.replace("RocblasMathModeSetter", "CompatRocblasMathModeSetter") - - # CURAND -> HIPRAND - s = s.replace("CURAND", "HIPRAND") - s = s.replace("Curand", "Hiprand") - s = s.replace("curand", "hiprand") - - # NCCL -> RCCL - # s = s.replace('NCCL_CALL', 'RCCL_CALL') - s = s.replace("#include ", "#include ") - - # CUDNN -> MIOpen - s = s.replace("CUDNN", "MIOPEN") - s = s.replace("Cudnn", "Miopen") - s = s.replace("cudnn", "miopen") - # hipify seems to have a bug for MIOpen, cudnn.h -> hipDNN.h, cudnn -> hipdnn - s = s.replace("#include ", "#include ") - s = s.replace("hipdnn", "miopen") - s = s.replace("HIPDNN_STATUS_SUCCESS", "miopenStatusSuccess") - s = s.replace("HIPDNN", "MIOPEN") - s = s.replace("MIOPEN_BATCHNORM_SPATIAL", "miopenBNSpatial") - s = s.replace("MIOPEN_BATCHNORM_PER_ACTIVATION", "miopenBNPerActivation") - s = s.replace("MIOPEN_LRN_CROSS_CHANNEL", "miopenLRNCrossChannel") - s = s.replace("MIOPEN_POOLING_MAX", "miopenPoolingMax") - s = s.replace("MIOPEN_POOLING_AVERAGE_COUNT_INCLUDE_PADDING", "miopenPoolingAverageInclusive") - s = s.replace("MIOPEN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING", "miopenPoolingAverage") - - # CUSPARSE -> HIPSPARSE - s = s.replace("CUSPARSE", "HIPSPARSE") - - # CUFFT -> HIPFFT - s = s.replace("CUFFT", "HIPFFT") - s = s.replace("cufftXtMakePlanMany", "hipfftXtMakePlanMany") - s = s.replace("cufftXtExec", "hipfftXtExec") - - # Undo where above hipify steps went too far. - s = s.replace("id, ROCM", "id, CUDA") # cuda_execution_provider.cc - s = s.replace("ROCM error executing", "HIP error executing") - s = s.replace("ROCM_PINNED", "CUDA_PINNED") - s = s.replace("rocm_err", "hip_err") - s = s.replace("RegisterHipTrainingKernels", "RegisterRocmTrainingKernels") - s = s.replace("ROCM_VERSION", "CUDA_VERSION") # semantically different meanings, cannot hipify - s = s.replace("__ROCM_ARCH__", "__CUDA_ARCH__") # semantically different meanings, cannot hipify - # "std::log" above incorrectly changed "std::logic_error" to "logfic_error" - s = s.replace("logfic_error", "std::logic_error") - - # Deletions - s = s.replace('#include "device_atomic_functions.h"', "") # HIP atomics in main hip header already - - # Fix warnings due to incorrect header paths, intentionally after all other hipify steps. - s = s.replace("#include ", "#include ") - s = s.replace("#include ", "#include ") - s = s.replace("#include ", "#include ") - s = s.replace("#include ", "#include ") - s = s.replace('#include "hipfft.h"', "#include ") - s = s.replace('#include "hipfftXt.h"', "#include ") - - # Fix onnxruntime/contrib_ops/rocm/transformers. They include cpu headers which use "cuda" in their names. - s = s.replace("rocm_device_prop_", "cuda_device_prop_") - s = s.replace("rocm_device_arch_", "cuda_device_arch_") - - s = s.replace("HipTuningContext", "RocmTuningContext") - - # We want hipfft, which needs hipDataType etc, but only do this for files that have "fft" in their names - # And we do this last, undoing or fixing hipify mistakes. - if "fft" in src_file_path: - s = s.replace("rocblas_datatype", "hipDataType") - s = s.replace("hipDataType_f32_c", "HIP_C_32F") - s = s.replace("hipDataType_f32_r", "HIP_R_32F") - s = s.replace("hipDataType_f64_c", "HIP_C_64F") - s = s.replace("hipDataType_f64_r", "HIP_R_64F") - s = s.replace("hipDataType_f16_c", "HIP_C_16F") - s = s.replace("hipDataType_f16_r", "HIP_R_16F") - - with open(dst_file_path, "w") as f: - f.write(s) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--hipify_perl", required=True) - parser.add_argument("--output", "-o", help="output file") - parser.add_argument("src", help="src") - args = parser.parse_args() - - hipify(args.hipify_perl, args.src, args.output) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 1fcd3fbe3daf0..ef76daeaa01bc 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -272,8 +272,6 @@ def generate_vcpkg_install_options(build_dir, args): vcpkg_install_options.append("--x-feature=qnn-ep") if args.use_rknpu: vcpkg_install_options.append("--x-feature=rknpu-ep") - if args.use_rocm: - vcpkg_install_options.append("--x-feature=rocm-ep") if args.use_tensorrt: vcpkg_install_options.append("--x-feature=tensorrt-ep") if args.use_vitisai: @@ -347,7 +345,6 @@ def generate_build_tree( build_dir, cuda_home, cudnn_home, - rocm_home, nccl_home, tensorrt_home, tensorrt_rtx_home, @@ -375,7 +372,7 @@ def generate_build_tree( # enable/disable float 8 types disable_float8_types = args.android or ("float8" in types_to_disable) # enable/disable float 4 type - disable_float4_types = args.android or args.use_rocm or ("float4" in types_to_disable) + disable_float4_types = args.android or ("float4" in types_to_disable) disable_optional_type = "optional" in types_to_disable disable_sparse_tensors = "sparsetensor" in types_to_disable if is_windows(): @@ -1492,20 +1489,6 @@ def setup_dml_build(args, cmake_path, build_dir, configs): raise BuildError("use_dml and minimal_build may not both be set") -def setup_rocm_build(args): - rocm_home = None - if args.use_rocm: - print(f"rocm_home = {args.rocm_home}") - rocm_home = args.rocm_home or None - rocm_home_not_valid = rocm_home and not os.path.exists(rocm_home) - if rocm_home_not_valid: - raise BuildError( - "rocm_home paths must be specified and valid.", - f"rocm_home='{rocm_home}' valid={rocm_home_not_valid}.", - ) - return rocm_home or "" - - def run_android_tests(args, source_dir, build_dir, config, cwd): if args.android_abi != "x86_64": log.info(f"--android_abi ({args.android_abi}) is not x86_64, skipping running of Android tests on emulator.") @@ -1760,12 +1743,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): if is_windows(): cwd = os.path.join(cwd, config) - if ( - not args.skip_pip_install - and args.enable_transformers_tool_test - and not args.disable_contrib_ops - and not args.use_rocm - ): + if not args.skip_pip_install and args.enable_transformers_tool_test and not args.disable_contrib_ops: # PyTorch is required for transformers tests, and optional for some python tests. # Install cpu only version of torch when cuda is not enabled in Linux. extra = [] if args.use_cuda and is_linux() else ["--index-url", "https://download.pytorch.org/whl/cpu"] @@ -1946,9 +1924,7 @@ def build_python_wheel( use_cuda, cuda_home, cuda_version, - use_rocm, use_migraphx, - rocm_version, use_dnnl, use_tensorrt, use_openvino, @@ -1994,12 +1970,6 @@ def build_python_wheel( cuda_version = cuda_version or parse_cuda_version_from_json(cuda_home) if cuda_version: args.append(f"--cuda_version={cuda_version}") - elif use_rocm: - args.append("--use_rocm") - if rocm_version: - args.append(f"--rocm_version={rocm_version}") - if use_migraphx: - args.append("--use_migraphx") elif use_migraphx: args.append("--use_migraphx") elif use_openvino: @@ -2039,7 +2009,6 @@ def build_nuget_package( build_dir, configs, use_cuda, - use_rocm, use_openvino, use_tensorrt, use_dnnl, @@ -2094,8 +2063,6 @@ def build_nuget_package( package_name = "/p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu" elif use_dml: package_name = "/p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML" - elif use_rocm: - package_name = "/p:OrtPackageId=Microsoft.ML.OnnxRuntime.ROCm" elif use_qnn: if use_qnn != "shared_lib": raise BuildError("Currently NuGet packages with QNN require QNN EP to be built as a shared library.") @@ -2447,9 +2414,6 @@ def main(): # if using migraphx, setup migraphx paths migraphx_home = setup_migraphx_vars(args) - # if using rocm, setup rocm paths - rocm_home = setup_rocm_build(args) - # if using cann, setup cann paths cann_home = setup_cann_vars(args) @@ -2572,16 +2536,12 @@ def main(): cwd=SCRIPT_DIR, ) - if args.use_rocm and args.rocm_version is None: - args.rocm_version = "" - generate_build_tree( cmake_path, source_dir, build_dir, cuda_home, cudnn_home, - rocm_home, nccl_home, tensorrt_home, tensorrt_rtx_home, @@ -2642,9 +2602,7 @@ def main(): args.use_cuda, cuda_home, args.cuda_version, - args.use_rocm, args.use_migraphx, - args.rocm_version, args.use_dnnl, args.use_tensorrt, args.use_openvino, @@ -2672,7 +2630,6 @@ def main(): build_dir, configs, args.use_cuda, - args.use_rocm, args.use_openvino, args.use_tensorrt, args.use_dnnl, diff --git a/tools/ci_build/build_args.py b/tools/ci_build/build_args.py index 6763973406294..cd652a6cbb82e 100644 --- a/tools/ci_build/build_args.py +++ b/tools/ci_build/build_args.py @@ -763,9 +763,6 @@ def add_execution_provider_args(parser: argparse.ArgumentParser) -> None: migx_group = parser.add_argument_group("MIGraphX Execution Provider") migx_group.add_argument("--use_migraphx", action="store_true", help="Enable MIGraphX EP.") migx_group.add_argument("--migraphx_home", help="Path to MIGraphX installation directory.") - migx_group.add_argument("--use_rocm", action="store_true", help="Enable ROCm EP.") - migx_group.add_argument("--rocm_version", help="ROCm stack version.") - migx_group.add_argument("--rocm_home", help="Path to ROCm installation directory.") # --- WebNN --- webnn_group = parser.add_argument_group("WebNN Execution Provider") diff --git a/tools/ci_build/gen_def.py b/tools/ci_build/gen_def.py index 526cc7bde519e..46cbac20627f7 100755 --- a/tools/ci_build/gen_def.py +++ b/tools/ci_build/gen_def.py @@ -71,7 +71,6 @@ def parse_arguments(): "vitisai", "winml", "cuda", - "rocm", "migraphx", "qnn", "snpe", diff --git a/tools/ci_build/github/pai/pai-excluded-tests.txt b/tools/ci_build/github/pai/pai-excluded-tests.txt deleted file mode 100644 index 845d36b71d215..0000000000000 --- a/tools/ci_build/github/pai/pai-excluded-tests.txt +++ /dev/null @@ -1,15 +0,0 @@ -CudaKernelTest.SoftmaxGrad_LargeTensor_LastAxis_Float16 -CudaKernelTest.SoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo -CudaKernelTest.SoftmaxGrad_LargeTensor_AllAxis_Float16 -CudaKernelTest.SoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo -CudaKernelTest.LogSoftmaxGrad_LargeTensor_LastAxis_Float16 -CudaKernelTest.LogSoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo -CudaKernelTest.LogSoftmaxGrad_LargeTensor_AllAxis_Float16 -CudaKernelTest.LogSoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo -ReductionOpTest.ReductionVariationTest -GatherOpTest.Gather_invalid_index_cpu -Scatter.InvalidIndex -GradientCheckerTest.AddGrad -GradientCheckerTest.SubGrad -GradientCheckerTest.MulGrad -GradientCheckerTest.DivGrad diff --git a/tools/ci_build/github/pai/pai_clean_device.sh b/tools/ci_build/github/pai/pai_clean_device.sh deleted file mode 100755 index 98b680d4f465c..0000000000000 --- a/tools/ci_build/github/pai/pai_clean_device.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -ex - -usage() { echo "Usage: $0 [-n ] [-d ] [-r ]" 1>&2; exit 1; } - -while getopts "n:d:r:" parameter_Option -do case "${parameter_Option}" -in -n) AGENT_NAME=${OPTARG};; -d) TARGET_DEVICE=${OPTARG};; -r) DRIVER_RENDER=${OPTARG};; -*) usage ;; -esac -done - -echo "Agent Name: $AGENT_NAME, Target Device: $TARGET_DEVICE, Driver Render: $DRIVER_RENDER" - -echo -e "\n ---- Execute rocm-smi" -rocm-smi - -echo -e "\n ---- Execute rocm-smi --showpids" -rocm-smi --showpids - -echo -e "\n ---- Execute rocm-smi --showpidgpus" -rocm-smi --showpidgpus - -echo -e "\n ---- Execute rocm-smi --showpids detail" -rocm-smi --showpids | awk '$1 ~/[0-9]+/{if((NR>6)) {print $1}}' | xargs -I {} ps {} - -echo -e "\n ---- Execute rocm-smi --showmeminfo" -rocm-smi --showmeminfo vram vis_vram gtt - -echo -e "\n ---- Clean up processes that use the target device $TARGET_DEVICE" -GPU_USED_BY_PIDS=$(rocm-smi --showpidgpus) -PID_NUMBERS_LINES=$(echo "$GPU_USED_BY_PIDS" | grep -n "DRM device" | cut -d ":" -f 1) -PID_NUMBERS_LINES_ARRAY=($PID_NUMBERS_LINES) - -for ((i = 0; i < ${#PID_NUMBERS_LINES_ARRAY[@]}; i++)); do - PID_NUMBER_LINE=${PID_NUMBERS_LINES_ARRAY[$i]} - PID_NUMBER=$(echo "$GPU_USED_BY_PIDS" | awk '{print $2}' | sed -n "${PID_NUMBER_LINE}p") - GPU_USED_BY_PID_LINE=$((PID_NUMBER_LINE + 1)) - GPU_USED_BY_PID=$(echo "$GPU_USED_BY_PIDS" | sed -n "${GPU_USED_BY_PID_LINE}p" | sed -e 's/^[ ]*//g' | sed -e 's/[ ]*$//g') - if [ "$GPU_USED_BY_PID" == "$TARGET_DEVICE" ]; then - echo "kill pid: $PID_NUMBER, using gpu: $GPU_USED_BY_PID" - kill -9 "$PID_NUMBER" - fi -done diff --git a/tools/ci_build/policheck_exclusions.xml b/tools/ci_build/policheck_exclusions.xml index a24eed809c5b8..9888245b48674 100644 --- a/tools/ci_build/policheck_exclusions.xml +++ b/tools/ci_build/policheck_exclusions.xml @@ -1,4 +1,4 @@ - LABELMAP.CS|OPERATORKERNELS.MD|BABEL.CONFIG.JS|METRO.CONFIG.JS|DMLOPERATORACTIVATION.CPP|DATA_OPS.CC|ONNX_CONVERTER.CC|ONNXOPS.PY|CPYTHON-PUBKEYS.TXT|AMD_HIPIFY.PY + LABELMAP.CS|OPERATORKERNELS.MD|BABEL.CONFIG.JS|METRO.CONFIG.JS|DMLOPERATORACTIVATION.CPP|DATA_OPS.CC|ONNX_CONVERTER.CC|ONNXOPS.PY|CPYTHON-PUBKEYS.TXT diff --git a/tools/ci_build/set-trigger-rules.py b/tools/ci_build/set-trigger-rules.py index 899aaaa95216a..98e3e6a9b05b2 100644 --- a/tools/ci_build/set-trigger-rules.py +++ b/tools/ci_build/set-trigger-rules.py @@ -22,7 +22,6 @@ "linux-migraphx-ci-pipeline.yml", "linux-openvino-ci-pipeline.yml", "linux-qnn-ci-pipeline.yml", - "linux-rocm-ci-pipeline.yml", "mac-ci-pipeline.yml", "mac-coreml-ci-pipeline.yml", "mac-ios-ci-pipeline.yml", diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index 6ce8c3b0bca91..9884cbf5793df 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -27,8 +27,6 @@ def get_package_name(os, cpu_arch, ep, is_training_package): pkg_name += "-cuda" elif ep == "tensorrt": pkg_name += "-tensorrt" - elif ep == "rocm": - pkg_name += "-rocm" elif ep == "migraphx": pkg_name += "-migraphx" elif os == "linux": @@ -38,8 +36,6 @@ def get_package_name(os, cpu_arch, ep, is_training_package): pkg_name += "-cuda" elif ep == "tensorrt": pkg_name += "-tensorrt" - elif ep == "rocm": - pkg_name += "-rocm" elif ep == "migraphx": pkg_name += "-migraphx" elif os == "osx": @@ -375,7 +371,6 @@ def generate_files(line_list, args): is_cuda_gpu_package = args.package_name == "Microsoft.ML.OnnxRuntime.Gpu" is_cuda_gpu_win_sub_package = args.package_name == "Microsoft.ML.OnnxRuntime.Gpu.Windows" is_cuda_gpu_linux_sub_package = args.package_name == "Microsoft.ML.OnnxRuntime.Gpu.Linux" - is_rocm_gpu_package = args.package_name == "Microsoft.ML.OnnxRuntime.ROCm" is_dml_package = args.package_name == "Microsoft.ML.OnnxRuntime.DirectML" is_windowsai_package = args.package_name == "Microsoft.AI.MachineLearning" is_snpe_package = args.package_name == "Microsoft.ML.OnnxRuntime.Snpe" @@ -440,7 +435,6 @@ def generate_files(line_list, args): "tensorrt_ep_shared_lib": "libonnxruntime_providers_tensorrt.so", "openvino_ep_shared_lib": "libonnxruntime_providers_openvino.so", "cuda_ep_shared_lib": "libonnxruntime_providers_cuda.so", - "rocm_ep_shared_lib": "libonnxruntime_providers_rocm.so", "migraphx_ep_shared_lib": "libonnxruntime_providers_migraphx.so", "onnxruntime_perf_test": "onnxruntime_perf_test", "onnx_test_runner": "onnx_test_runner", @@ -631,8 +625,6 @@ def generate_files(line_list, args): # downloaded from other build jobs if is_cuda_gpu_package or is_cuda_gpu_win_sub_package or is_cuda_gpu_linux_sub_package: ep_list = ["tensorrt", "cuda", None] - elif is_rocm_gpu_package: - ep_list = ["rocm", None] elif is_migraphx_package: ep_list = ["migraphx", None] else: @@ -742,24 +734,6 @@ def generate_files(line_list, args): + '\\native" />' ) - if args.execution_provider == "rocm" or (is_rocm_gpu_package and not is_ado_packaging_build): - files_list.append( - "' - ) - files_list.append( - "' - ) - if args.execution_provider == "openvino": openvino_path = get_env_var("INTEL_OPENVINO_DIR") files_list.append( @@ -998,7 +972,6 @@ def _files_list_append(key: str): or is_cuda_gpu_package or is_cuda_gpu_linux_sub_package or is_cuda_gpu_win_sub_package - or is_rocm_gpu_package or is_migraphx_package or is_dml_package or is_mklml_package @@ -1240,12 +1213,11 @@ def validate_execution_provider(execution_provider): or execution_provider == "cuda" or execution_provider == "tensorrt" or execution_provider == "openvino" - or execution_provider == "rocm" or execution_provider == "migraphx" ): raise Exception( "On Linux platform nuget generation is supported only " - "for cpu|cuda|dnnl|tensorrt|openvino|rocm execution providers." + "for cpu|cuda|dnnl|tensorrt|openvino|migraphx execution providers." )