diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 8fdbca776745f..1b60ea655a2d8 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -18,9 +18,9 @@ set(mlas_common_srcs if(MSVC) if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64") - set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/sgemma.asm) - set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.i) - set(obj_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.obj) + set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/SgemmKernelNeon.asm) + set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/SgemmKernelNeon.i) + set(obj_filename ${CMAKE_CURRENT_BINARY_DIR}/SgemmKernelNeon.obj) if(CMAKE_BUILD_TYPE STREQUAL "Debug") set(ARMASM_FLAGS "-g") @@ -78,7 +78,7 @@ else() if (CMAKE_ANDROID_ARCH_ABI STREQUAL "armeabi-v7a") set(ARM TRUE) elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "arm64-v8a") - set(ARM TRUE) # Android NDK fails to compile sgemma.s + set(ARM64 TRUE) elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "x86_64") set(X86_64 TRUE) elseif (CMAKE_ANDROID_ARCH_ABI STREQUAL "x86") @@ -111,7 +111,7 @@ else() enable_language(ASM) set(mlas_platform_srcs - ${ONNXRUNTIME_ROOT}/core/mlas/lib/aarch64/sgemma.s + ${ONNXRUNTIME_ROOT}/core/mlas/lib/aarch64/SgemmKernelNeon.S ) elseif(X86) enable_language(ASM) diff --git a/onnxruntime/core/mlas/lib/aarch64/sgemma.s b/onnxruntime/core/mlas/lib/aarch64/SgemmKernelNeon.S similarity index 95% rename from onnxruntime/core/mlas/lib/aarch64/sgemma.s rename to onnxruntime/core/mlas/lib/aarch64/SgemmKernelNeon.S index 545465a5a86e8..c69fadc36893b 100644 --- a/onnxruntime/core/mlas/lib/aarch64/sgemma.s +++ b/onnxruntime/core/mlas/lib/aarch64/SgemmKernelNeon.S @@ -6,7 +6,7 @@ Licensed under the MIT License. Module Name: - sgemma.s + SgemmKernelNeon.s Abstract: @@ -88,7 +88,7 @@ Abstract: .endm - +// // MultiplyAccumulateRow // // Generates the code to multiply and accumulate a single row of the output @@ -137,11 +137,11 @@ Abstract: ClearBlockAccumulators \Columns\(),\Rows\() .if \Rows\() >= 2 - add x10,x0,x6,uxtw 2 // compute matrix A plus 1 row + add x10,x0,x6,lsl #2 // compute matrix A plus 1 row .endif .if \Rows\() >= 4 - add x11,x10,x6,uxtw 2 // compute matrix A plus 2 rows - add x12,x11,x6,uxtw 2 // compute matrix A plus 3 rows + add x11,x10,x6,lsl #2 // compute matrix A plus 2 rows + add x12,x11,x6,lsl #2 // compute matrix A plus 3 rows .endif sub x9,x3,#4 // decrement block count to process @@ -183,7 +183,7 @@ Abstract: ldp q6,q7,[x1,#-8*4] .endif MultiplyAccumulateBlock \Columns\(),\Rows\(),0 - sub x9,x9,1 + sub x9,x9,#1 cbnz x9,.L\Mode\().Compute\Columns\().x\Rows\().BlockBy1Loop .L\Mode\().Output\Columns\().x\Rows\().Block: @@ -430,12 +430,12 @@ Return Value: .type MlasSgemmKernel\Mode\(),%function MlasSgemmKernel\Mode\(): - stp d8,d9,[sp,-32]! - stp d10,d11,[sp,16] + stp d8,d9,[sp,#-32]! + stp d10,d11,[sp,#16] - add x13,x2,x7,uxtw 2 // compute matrix C plus 1 row - add x14,x13,x7,uxtw 2 // compute matrix C plus 2 rows - add x15,x14,x7,uxtw 2 // compute matrix C plus 3 rows + add x13,x2,x7,lsl #2 // compute matrix C plus 1 row + add x14,x13,x7,lsl #2 // compute matrix C plus 2 rows + add x15,x14,x7,lsl #2 // compute matrix C plus 3 rows mov x8,x0 // save matrix A // @@ -452,8 +452,8 @@ MlasSgemmKernel\Mode\(): .L\Mode\().ExitKernel: mov x0,x4 - ldp d10,d11,[sp,16] - ldp d8,d9,[sp],32 + ldp d10,d11,[sp,#16] + ldp d8,d9,[sp],#32 ret // diff --git a/onnxruntime/core/mlas/lib/arm64/sgemma.asm b/onnxruntime/core/mlas/lib/arm64/SgemmKernelNeon.asm similarity index 91% rename from onnxruntime/core/mlas/lib/arm64/sgemma.asm rename to onnxruntime/core/mlas/lib/arm64/SgemmKernelNeon.asm index 0b6eb11fa2d78..3675689db6cd5 100644 --- a/onnxruntime/core/mlas/lib/arm64/sgemma.asm +++ b/onnxruntime/core/mlas/lib/arm64/SgemmKernelNeon.asm @@ -6,7 +6,7 @@ ; ; Module Name: ; -; sgemma.asm +; SgemmKernelNeon.asm ; ; Abstract: ; @@ -19,31 +19,6 @@ TEXTAREA -; -; ComputeEffectiveAddress -; -; Generates the code to compute the effective address of a matrix element using -; the instruction template: -; -; add $DestReg,$BaseReg,$IndexReg lsl #2 -; -; For native ARM64, the macro generates a 64-bit address calculation. For CHPE -; targets, the macro generates a 32-bit address calculation to stay within the -; WOW64 sandbox. -; - - - MACRO - ComputeEffectiveAddress $DestReg, $BaseReg, $IndexReg - -#if defined(_CHPE_X86_ARM64_) - DCD 0x0B000800:OR:(:RCONST:$DestReg):OR:((:RCONST:$BaseReg):SHL:5):OR:((:RCONST:$IndexReg):SHL:16) -#else - DCD 0x8B000800:OR:(:RCONST:$DestReg):OR:((:RCONST:$BaseReg):SHL:5):OR:((:RCONST:$IndexReg):SHL:16) -#endif - - MEND - ; ; ClearRowAccumulators ; @@ -171,11 +146,11 @@ ClearBlockAccumulators $Columns, $Rows IF $Rows >= 2 - ComputeEffectiveAddress x10,x0,x6 ; compute matrix A plus 1 row + add x10,x0,x6 lsl #2 ; compute matrix A plus 1 row ENDIF IF $Rows >= 4 - ComputeEffectiveAddress x11,x10,x6 ; compute matrix A plus 2 rows - ComputeEffectiveAddress x12,x11,x6 ; compute matrix A plus 3 rows + add x11,x10,x6 lsl #2 ; compute matrix A plus 2 rows + add x12,x11,x6 lsl #2 ; compute matrix A plus 3 rows ENDIF sub x9,x3,#4 ; decrement block count to process @@ -217,7 +192,7 @@ $Mode.Compute$Columns.x$Rows.BlockBy1Loop ldp v6,v7,[x1,#-8*4] ENDIF MultiplyAccumulateBlock $Columns,$Rows,0 - sub x9,x9,1 + sub x9,x9,#1 cbnz x9,$Mode.Compute$Columns.x$Rows.BlockBy1Loop $Mode.Output$Columns.x$Rows.Block @@ -476,9 +451,9 @@ $Mode.OutputRemaining1x$Rows.Block PROLOG_SAVE_REG_PAIR d8,d9,#-32! PROLOG_SAVE_REG_PAIR d10,d11,#16 - ComputeEffectiveAddress x13,x2,x7 ; compute matrix C plus 1 row - ComputeEffectiveAddress x14,x13,x7 ; compute matrix C plus 2 rows - ComputeEffectiveAddress x15,x14,x7 ; compute matrix C plus 3 rows + add x13,x2,x7 lsl #2 ; compute matrix C plus 1 row + add x14,x13,x7 lsl #2 ; compute matrix C plus 2 rows + add x15,x14,x7 lsl #2 ; compute matrix C plus 3 rows mov x8,x0 ; save matrix A ; diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 40150e5f44beb..1ae227d1f9ac1 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -80,7 +80,7 @@ Module Name: #if defined(_M_AMD64) || defined(__x86_64__) #define MLAS_TARGET_AMD64 #endif -#if (defined(_M_IX86) && !defined(_M_HYBRID_X86_ARM64)) || defined(__i386__) +#if defined(_M_IX86) || defined(__i386__) #define MLAS_TARGET_IX86 #endif #if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_IX86) @@ -560,7 +560,7 @@ MlasGetMaximumThreadCount( #if defined(MLAS_TARGET_ARM) #define MLAS_NEON_INTRINSICS #define MLAS_NEON32_INTRINSICS -#elif defined(MLAS_TARGET_ARM64) || defined(_M_HYBRID_X86_ARM64) +#elif defined(MLAS_TARGET_ARM64) #define MLAS_NEON_INTRINSICS #define MLAS_NEON64_INTRINSICS #elif defined(MLAS_TARGET_AMD64_IX86)