diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 75b8e736251d8b..1448cf7dea9c62 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -108,6 +108,16 @@ static cl::opt UsePrecSqrtF32( cl::desc("NVPTX Specific: 0 use sqrt.approx, 1 use sqrt.rn."), cl::init(true)); +// PTX atom.add.f32 has fixed FTZ behavior that may not match the function's +// (see shouldExpandAtomicRMWInIR), so by default we fall back to a CAS loop +// when they disagree. This flag is an escape hatch to use atom.add anyway, +// trading correct denormal handling for the speed of the native instruction. +static cl::opt AllowFTZAtomics( + "nvptx-allow-ftz-atomics", cl::Hidden, + cl::desc("NVPTX Specific: Lower atomicrmw fadd to atom.add even when its " + "FTZ behavior does not match the function's denormal mode."), + cl::init(false)); + /// Whereas CUDA's implementation (see libdevice) uses ex2.approx for exp2(), it /// does NOT use lg2.approx for log2, so this is disabled by default. static cl::opt UseApproxLog2F32( @@ -7466,22 +7476,54 @@ NVPTXTargetLowering::AtomicExpansionKind NVPTXTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const { Type *Ty = AI->getValOperand()->getType(); - if (AI->isFloatingPointOperation()) { - if (AI->getOperation() == AtomicRMWInst::BinOp::FAdd) { - if (Ty->isHalfTy() && STI.getSmVersion() >= 70 && - STI.getPTXVersion() >= 63) - return AtomicExpansionKind::None; - if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 && - STI.getPTXVersion() >= 78) - return AtomicExpansionKind::None; - if (Ty->isFloatTy()) - return AtomicExpansionKind::None; - if (Ty->isDoubleTy() && STI.hasAtomAddF64()) + // Try to lower LLVM atomicrmw fadd to PTX atomic.add. This is complicated + // by the weird FTZ behavior PTX atom.add has: + // - atom.add.f32 on global memory flushes denormals + // - atom.add.f32 on shared memory does not flush denormals + // - atom.add.f16 and atomic.add.bf16 never flush denormals + // + // We lower to atom.add only if the function's FTZ behavior matches that of + // atom.add; otherwise, we lower to a CAS loop. But we always allow + // atomic.add.bf16; even though it never flushes denormals, we never flush + // bf16 denormals when doing regular arithmetic, even when FTZ is enabled. + if (AI->isFloatingPointOperation() && + AI->getOperation() == AtomicRMWInst::BinOp::FAdd) { + const bool FTZ = + AI->getFunction()->getDenormalMode(APFloat::IEEEsingle()).Output == + DenormalMode::PreserveSign; + + // AllowFTZAtomics forces atom.add regardless of the FTZ mismatch. + if (Ty->isFloatTy()) { + bool UseNative = AllowFTZAtomics; + switch (AI->getPointerAddressSpace()) { + case llvm::ADDRESS_SPACE_GLOBAL: + UseNative |= FTZ; + break; + case llvm::ADDRESS_SPACE_SHARED: + case llvm::ADDRESS_SPACE_SHARED_CLUSTER: + UseNative |= !FTZ; + break; + } + if (UseNative) return AtomicExpansionKind::None; } - return AtomicExpansionKind::CmpXChg; + + if (Ty->isHalfTy() && (!FTZ || AllowFTZAtomics) && + STI.getSmVersion() >= 70 && STI.getPTXVersion() >= 63) + return AtomicExpansionKind::None; + + if (Ty->isBFloatTy() && STI.getSmVersion() >= 90 && + STI.getPTXVersion() >= 78) + return AtomicExpansionKind::None; + + if (Ty->isDoubleTy() && STI.hasAtomAddF64()) + return AtomicExpansionKind::None; } + // PTX's only atomic fp op is `add`; all other ops expand to a CAS loop. + if (AI->isFloatingPointOperation()) + return AtomicExpansionKind::CmpXChg; + assert(Ty->isIntegerTy() && "Ty should be integer at this point"); const unsigned BitWidth = cast(Ty)->getBitWidth(); diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll new file mode 100644 index 00000000000000..727c13bb4e47e7 --- /dev/null +++ b/llvm/test/CodeGen/NVPTX/atomicrmw-allow-ftz-atomics.ll @@ -0,0 +1,105 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; -nvptx-allow-ftz-atomics forces atomicrmw fadd to lower to atom.add even when +; the hardware's FTZ behavior doesn't match the function's denormal mode, where +; we'd otherwise emit a CAS loop. Compare the default lowering (DEFAULT) against +; the flag (ALLOW). +; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=CHECK,DEFAULT +; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | FileCheck %s --check-prefixes=CHECK,ALLOW +; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %} +; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -nvptx-allow-ftz-atomics | %ptxas-verify -arch=sm_90 %} + +; global f32: atom.add flushes, so default uses a CAS loop (function is not FTZ); +; the flag forces atom.add. +define float @fadd_f32_global(ptr addrspace(1) %addr, float %val) { +; DEFAULT-LABEL: fadd_f32_global( +; DEFAULT: { +; DEFAULT-NEXT: .reg .pred %p<2>; +; DEFAULT-NEXT: .reg .b32 %r<5>; +; DEFAULT-NEXT: .reg .b64 %rd<2>; +; DEFAULT-EMPTY: +; DEFAULT-NEXT: // %bb.0: +; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_global_param_1]; +; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0]; +; DEFAULT-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1]; +; DEFAULT-NEXT: $L__BB0_1: // %atomicrmw.start +; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1 +; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2; +; DEFAULT-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3; +; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4; +; DEFAULT-NEXT: mov.b32 %r4, %r1; +; DEFAULT-NEXT: @%p1 bra $L__BB0_1; +; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end +; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1; +; DEFAULT-NEXT: ret; +; +; ALLOW-LABEL: fadd_f32_global( +; ALLOW: { +; ALLOW-NEXT: .reg .b32 %r<3>; +; ALLOW-NEXT: .reg .b64 %rd<2>; +; ALLOW-EMPTY: +; ALLOW-NEXT: // %bb.0: +; ALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_global_param_0]; +; ALLOW-NEXT: ld.param.b32 %r1, [fadd_f32_global_param_1]; +; ALLOW-NEXT: atom.relaxed.sys.global.add.f32 %r2, [%rd1], %r1; +; ALLOW-NEXT: st.param.b32 [func_retval0], %r2; +; ALLOW-NEXT: ret; + %r = atomicrmw fadd ptr addrspace(1) %addr, float %val monotonic + ret float %r +} + +; generic f32: we can't know the memory space, so default always uses a CAS loop; +; the flag forces atom.add. +define float @fadd_f32_generic(ptr %addr, float %val) { +; DEFAULT-LABEL: fadd_f32_generic( +; DEFAULT: { +; DEFAULT-NEXT: .reg .pred %p<2>; +; DEFAULT-NEXT: .reg .b32 %r<5>; +; DEFAULT-NEXT: .reg .b64 %rd<2>; +; DEFAULT-EMPTY: +; DEFAULT-NEXT: // %bb.0: +; DEFAULT-NEXT: ld.param.b32 %r2, [fadd_f32_generic_param_1]; +; DEFAULT-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0]; +; DEFAULT-NEXT: ld.relaxed.sys.b32 %r4, [%rd1]; +; DEFAULT-NEXT: $L__BB1_1: // %atomicrmw.start +; DEFAULT-NEXT: // =>This Inner Loop Header: Depth=1 +; DEFAULT-NEXT: add.rn.f32 %r3, %r4, %r2; +; DEFAULT-NEXT: atom.relaxed.sys.cas.b32 %r1, [%rd1], %r4, %r3; +; DEFAULT-NEXT: setp.ne.b32 %p1, %r1, %r4; +; DEFAULT-NEXT: mov.b32 %r4, %r1; +; DEFAULT-NEXT: @%p1 bra $L__BB1_1; +; DEFAULT-NEXT: // %bb.2: // %atomicrmw.end +; DEFAULT-NEXT: st.param.b32 [func_retval0], %r1; +; DEFAULT-NEXT: ret; +; +; ALLOW-LABEL: fadd_f32_generic( +; ALLOW: { +; ALLOW-NEXT: .reg .b32 %r<3>; +; ALLOW-NEXT: .reg .b64 %rd<2>; +; ALLOW-EMPTY: +; ALLOW-NEXT: // %bb.0: +; ALLOW-NEXT: ld.param.b64 %rd1, [fadd_f32_generic_param_0]; +; ALLOW-NEXT: ld.param.b32 %r1, [fadd_f32_generic_param_1]; +; ALLOW-NEXT: atom.relaxed.sys.add.f32 %r2, [%rd1], %r1; +; ALLOW-NEXT: st.param.b32 [func_retval0], %r2; +; ALLOW-NEXT: ret; + %r = atomicrmw fadd ptr %addr, float %val monotonic + ret float %r +} + +; shared f32: atom.add does not flush, matching the non-FTZ function, so this is +; atom.add either way. +define float @fadd_f32_shared(ptr addrspace(3) %addr, float %val) { +; CHECK-LABEL: fadd_f32_shared( +; CHECK: { +; CHECK-NEXT: .reg .b32 %r<3>; +; CHECK-NEXT: .reg .b64 %rd<2>; +; CHECK-EMPTY: +; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.b64 %rd1, [fadd_f32_shared_param_0]; +; CHECK-NEXT: ld.param.b32 %r1, [fadd_f32_shared_param_1]; +; CHECK-NEXT: atom.relaxed.sys.shared.add.f32 %r2, [%rd1], %r1; +; CHECK-NEXT: st.param.b32 [func_retval0], %r2; +; CHECK-NEXT: ret; + %r = atomicrmw fadd ptr addrspace(3) %addr, float %val monotonic + ret float %r +} diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll index 8886873fd5c2eb..5eb508eb12e176 100644 --- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll +++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm60.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 -; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefix=SM60 +; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | FileCheck %s --check-prefixes=SM60,SM60-NOFTZ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM60,SM60-FTZ ; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 | %ptxas-verify -arch=sm_60 %} +; RUN: %if ptxas-sm_60 && ptxas-isa-5.0 %{ llc < %s -march=nvptx64 -mcpu=sm_60 -mattr=+ptx50 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_60 %} define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM60-LABEL: xchg_acq_rel_i8_global_cta( @@ -1737,168 +1739,322 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB60_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .b32 %r<3>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fsub_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b32 %r<5>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1]; -; SM60-NEXT: $L__BB61_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: sub.rn.f32 %r3, %r4, %r2; -; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM60-NEXT: mov.b32 %r4, %r1; -; SM60-NEXT: @%p1 bra $L__BB61_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b32 [func_retval0], %r1; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB61_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<5>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-FTZ-NEXT: mov.b32 %r4, %r1; +; SM60-FTZ-NEXT: @%p1 bra $L__BB61_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fmin_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b32 %r<5>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1]; -; SM60-NEXT: $L__BB62_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: min.f32 %r3, %r4, %r2; -; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM60-NEXT: mov.b32 %r4, %r1; -; SM60-NEXT: @%p1 bra $L__BB62_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b32 [func_retval0], %r1; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: min.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB62_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<5>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-FTZ-NEXT: mov.b32 %r4, %r1; +; SM60-FTZ-NEXT: @%p1 bra $L__BB62_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fmax_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b32 %r<5>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: ld.volatile.global.b32 %r4, [%rd1]; -; SM60-NEXT: $L__BB63_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: max.f32 %r3, %r4, %r2; -; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM60-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM60-NEXT: mov.b32 %r4, %r1; -; SM60-NEXT: @%p1 bra $L__BB63_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b32 [func_retval0], %r1; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: max.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB63_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<5>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-FTZ-NEXT: mov.b32 %r4, %r1; +; SM60-FTZ-NEXT: @%p1 bra $L__BB63_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fminimum_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<6>; -; SM60-NEXT: .reg .b32 %r<9>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1]; -; SM60-NEXT: setp.eq.b32 %p3, %r2, -2147483648; -; SM60-NEXT: $L__BB64_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: setp.nan.f32 %p1, %r8, %r2; -; SM60-NEXT: min.f32 %r3, %r8, %r2; -; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; -; SM60-NEXT: setp.eq.b32 %p2, %r8, -2147483648; -; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2; -; SM60-NEXT: selp.f32 %r6, %r2, %r5, %p3; -; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; -; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4; -; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; -; SM60-NEXT: setp.ne.b32 %p5, %r1, %r8; -; SM60-NEXT: mov.b32 %r8, %r1; -; SM60-NEXT: @%p5 bra $L__BB64_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b32 [func_retval0], %r1; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<6>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<9>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1]; +; SM60-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648; +; SM60-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2; +; SM60-NOFTZ-NEXT: min.f32 %r3, %r8, %r2; +; SM60-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM60-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648; +; SM60-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM60-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; +; SM60-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM60-NOFTZ-NEXT: mov.b32 %r8, %r1; +; SM60-NOFTZ-NEXT: @%p5 bra $L__BB64_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<6>; +; SM60-FTZ-NEXT: .reg .b32 %r<9>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1]; +; SM60-FTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648; +; SM60-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2; +; SM60-FTZ-NEXT: min.ftz.f32 %r3, %r8, %r2; +; SM60-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM60-FTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648; +; SM60-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM60-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000; +; SM60-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM60-FTZ-NEXT: mov.b32 %r8, %r1; +; SM60-FTZ-NEXT: @%p5 bra $L__BB64_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fmaximum_acq_rel_float_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<6>; -; SM60-NEXT: .reg .b32 %r<9>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: ld.volatile.global.b32 %r8, [%rd1]; -; SM60-NEXT: setp.eq.b32 %p3, %r2, 0; -; SM60-NEXT: $L__BB65_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: setp.nan.f32 %p1, %r8, %r2; -; SM60-NEXT: max.f32 %r3, %r8, %r2; -; SM60-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; -; SM60-NEXT: setp.eq.b32 %p2, %r8, 0; -; SM60-NEXT: selp.f32 %r5, %r8, %r4, %p2; -; SM60-NEXT: selp.f32 %r6, %r2, %r5, %p3; -; SM60-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; -; SM60-NEXT: selp.f32 %r7, %r6, %r4, %p4; -; SM60-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; -; SM60-NEXT: setp.ne.b32 %p5, %r1, %r8; -; SM60-NEXT: mov.b32 %r8, %r1; -; SM60-NEXT: @%p5 bra $L__BB65_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b32 [func_retval0], %r1; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<6>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<9>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1]; +; SM60-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, 0; +; SM60-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2; +; SM60-NOFTZ-NEXT: max.f32 %r3, %r8, %r2; +; SM60-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM60-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, 0; +; SM60-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM60-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM60-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; +; SM60-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM60-NOFTZ-NEXT: mov.b32 %r8, %r1; +; SM60-NOFTZ-NEXT: @%p5 bra $L__BB65_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<6>; +; SM60-FTZ-NEXT: .reg .b32 %r<9>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r8, [%rd1]; +; SM60-FTZ-NEXT: setp.eq.b32 %p3, %r2, 0; +; SM60-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2; +; SM60-FTZ-NEXT: max.ftz.f32 %r3, %r8, %r2; +; SM60-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM60-FTZ-NEXT: setp.eq.b32 %p2, %r8, 0; +; SM60-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM60-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000; +; SM60-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM60-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM60-FTZ-NEXT: mov.b32 %r8, %r1; +; SM60-FTZ-NEXT: @%p5 bra $L__BB65_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -2065,591 +2221,1133 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double } define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fadd_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b16 %rs<4>; -; SM60-NEXT: .reg .b32 %r<15>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1]; -; SM60-NEXT: $L__BB72_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r14, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: add.rn.f16 %rs3, %rs2, %rs1; -; SM60-NEXT: cvt.u32.u16 %r9, %rs3; -; SM60-NEXT: shl.b32 %r10, %r9, %r1; -; SM60-NEXT: and.b32 %r11, %r14, %r2; -; SM60-NEXT: or.b32 %r12, %r11, %r10; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM60-NEXT: mov.b32 %r14, %r3; -; SM60-NEXT: @%p1 bra $L__BB72_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r13, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r13; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB72_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: add.rn.f16 %rs3, %rs2, %rs1; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB72_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b16 %rs<4>; +; SM60-FTZ-NEXT: .reg .b32 %r<15>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM60-FTZ-NEXT: mov.b32 %r14, %r3; +; SM60-FTZ-NEXT: @%p1 bra $L__BB72_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fsub_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b16 %rs<4>; -; SM60-NEXT: .reg .b32 %r<15>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1]; -; SM60-NEXT: $L__BB73_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r14, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; -; SM60-NEXT: cvt.u32.u16 %r9, %rs3; -; SM60-NEXT: shl.b32 %r10, %r9, %r1; -; SM60-NEXT: and.b32 %r11, %r14, %r2; -; SM60-NEXT: or.b32 %r12, %r11, %r10; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM60-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM60-NEXT: mov.b32 %r14, %r3; -; SM60-NEXT: @%p1 bra $L__BB73_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r13, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r13; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB73_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b16 %rs<4>; +; SM60-FTZ-NEXT: .reg .b32 %r<15>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM60-FTZ-NEXT: mov.b32 %r14, %r3; +; SM60-FTZ-NEXT: @%p1 bra $L__BB73_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fmin_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b16 %rs<4>; -; SM60-NEXT: .reg .b32 %r<18>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1]; -; SM60-NEXT: cvt.f32.f16 %r10, %rs1; -; SM60-NEXT: $L__BB74_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r17, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: cvt.f32.f16 %r9, %rs2; -; SM60-NEXT: min.f32 %r11, %r9, %r10; -; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11; -; SM60-NEXT: cvt.u32.u16 %r12, %rs3; -; SM60-NEXT: shl.b32 %r13, %r12, %r1; -; SM60-NEXT: and.b32 %r14, %r17, %r2; -; SM60-NEXT: or.b32 %r15, %r14, %r13; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; -; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17; -; SM60-NEXT: mov.b32 %r17, %r3; -; SM60-NEXT: @%p1 bra $L__BB74_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r16, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r16; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<18>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1; +; SM60-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2; +; SM60-NOFTZ-NEXT: min.f32 %r11, %r9, %r10; +; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB74_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b16 %rs<4>; +; SM60-FTZ-NEXT: .reg .b32 %r<18>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1]; +; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1; +; SM60-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2; +; SM60-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10; +; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM60-FTZ-NEXT: mov.b32 %r17, %r3; +; SM60-FTZ-NEXT: @%p1 bra $L__BB74_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fmax_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<2>; -; SM60-NEXT: .reg .b16 %rs<4>; -; SM60-NEXT: .reg .b32 %r<18>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r17, [%rd1]; -; SM60-NEXT: cvt.f32.f16 %r10, %rs1; -; SM60-NEXT: $L__BB75_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r17, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: cvt.f32.f16 %r9, %rs2; -; SM60-NEXT: max.f32 %r11, %r9, %r10; -; SM60-NEXT: cvt.rn.f16.f32 %rs3, %r11; -; SM60-NEXT: cvt.u32.u16 %r12, %rs3; -; SM60-NEXT: shl.b32 %r13, %r12, %r1; -; SM60-NEXT: and.b32 %r14, %r17, %r2; -; SM60-NEXT: or.b32 %r15, %r14, %r13; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; -; SM60-NEXT: setp.ne.b32 %p1, %r3, %r17; -; SM60-NEXT: mov.b32 %r17, %r3; -; SM60-NEXT: @%p1 bra $L__BB75_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r16, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r16; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<18>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1; +; SM60-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2; +; SM60-NOFTZ-NEXT: max.f32 %r11, %r9, %r10; +; SM60-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM60-NOFTZ-NEXT: mov.b32 %r17, %r3; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB75_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b16 %rs<4>; +; SM60-FTZ-NEXT: .reg .b32 %r<18>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r17, [%rd1]; +; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1; +; SM60-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2; +; SM60-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10; +; SM60-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM60-FTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM60-FTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM60-FTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM60-FTZ-NEXT: mov.b32 %r17, %r3; +; SM60-FTZ-NEXT: @%p1 bra $L__BB75_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fminimum_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<7>; -; SM60-NEXT: .reg .b16 %rs<9>; -; SM60-NEXT: .reg .b32 %r<15>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1]; -; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768; -; SM60-NEXT: $L__BB76_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r14, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: setp.lt.f16 %p1, %rs2, %rs1; -; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1; -; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; -; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768; -; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM60-NEXT: mov.b16 %rs7, 0x0000; -; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7; -; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; -; SM60-NEXT: cvt.u32.u16 %r9, %rs8; -; SM60-NEXT: shl.b32 %r10, %r9, %r1; -; SM60-NEXT: and.b32 %r11, %r14, %r2; -; SM60-NEXT: or.b32 %r12, %r11, %r10; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14; -; SM60-NEXT: mov.b32 %r14, %r3; -; SM60-NEXT: @%p6 bra $L__BB76_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r13, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r13; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<7>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<9>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM60-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: setp.lt.f16 %p1, %rs2, %rs1; +; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1; +; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-NOFTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM60-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7; +; SM60-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM60-NOFTZ-NEXT: @%p6 bra $L__BB76_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<7>; +; SM60-FTZ-NEXT: .reg .b16 %rs<9>; +; SM60-FTZ-NEXT: .reg .b32 %r<15>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM60-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: setp.lt.ftz.f16 %p1, %rs2, %rs1; +; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1; +; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-FTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7; +; SM60-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM60-FTZ-NEXT: mov.b32 %r14, %r3; +; SM60-FTZ-NEXT: @%p6 bra $L__BB76_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM60-LABEL: fmaximum_acq_rel_half_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<7>; -; SM60-NEXT: .reg .b16 %rs<9>; -; SM60-NEXT: .reg .b32 %r<15>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r14, [%rd1]; -; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0; -; SM60-NEXT: $L__BB77_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r14, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: setp.gt.f16 %p1, %rs2, %rs1; -; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM60-NEXT: setp.nan.f16 %p2, %rs2, %rs1; -; SM60-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; -; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0; -; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM60-NEXT: mov.b16 %rs7, 0x0000; -; SM60-NEXT: setp.eq.f16 %p5, %rs4, %rs7; -; SM60-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; -; SM60-NEXT: cvt.u32.u16 %r9, %rs8; -; SM60-NEXT: shl.b32 %r10, %r9, %r1; -; SM60-NEXT: and.b32 %r11, %r14, %r2; -; SM60-NEXT: or.b32 %r12, %r11, %r10; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM60-NEXT: setp.ne.b32 %p6, %r3, %r14; -; SM60-NEXT: mov.b32 %r14, %r3; -; SM60-NEXT: @%p6 bra $L__BB77_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r13, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r13; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<7>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<9>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM60-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: setp.gt.f16 %p1, %rs2, %rs1; +; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1; +; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-NOFTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM60-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7; +; SM60-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM60-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM60-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM60-NOFTZ-NEXT: @%p6 bra $L__BB77_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<7>; +; SM60-FTZ-NEXT: .reg .b16 %rs<9>; +; SM60-FTZ-NEXT: .reg .b32 %r<15>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r14, [%rd1]; +; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM60-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: setp.gt.ftz.f16 %p1, %rs2, %rs1; +; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1; +; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-FTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM60-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7; +; SM60-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM60-FTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM60-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM60-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM60-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM60-FTZ-NEXT: mov.b32 %r14, %r3; +; SM60-FTZ-NEXT: @%p6 bra $L__BB77_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fadd_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<3>; -; SM60-NEXT: .reg .b16 %rs<2>; -; SM60-NEXT: .reg .b32 %r<24>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: $L__BB78_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r23, %r1; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: add.rn.f32 %r12, %r9, %r11; -; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM60-NEXT: add.s32 %r14, %r13, %r12; -; SM60-NEXT: add.s32 %r15, %r14, 32767; -; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM60-NEXT: or.b32 %r16, %r12, 4194304; -; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM60-NEXT: shr.u32 %r18, %r17, 16; -; SM60-NEXT: shl.b32 %r19, %r18, %r1; -; SM60-NEXT: and.b32 %r20, %r23, %r2; -; SM60-NEXT: or.b32 %r21, %r20, %r19; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM60-NEXT: mov.b32 %r23, %r3; -; SM60-NEXT: @%p2 bra $L__BB78_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r22, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r22; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<3>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11; +; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM60-NOFTZ-NEXT: @%p2 bra $L__BB78_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<3>; +; SM60-FTZ-NEXT: .reg .b16 %rs<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<24>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11; +; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-FTZ-NEXT: mov.b32 %r23, %r3; +; SM60-FTZ-NEXT: @%p2 bra $L__BB78_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fsub_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<3>; -; SM60-NEXT: .reg .b16 %rs<2>; -; SM60-NEXT: .reg .b32 %r<24>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: $L__BB79_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r23, %r1; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: sub.rn.f32 %r12, %r9, %r11; -; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM60-NEXT: add.s32 %r14, %r13, %r12; -; SM60-NEXT: add.s32 %r15, %r14, 32767; -; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM60-NEXT: or.b32 %r16, %r12, 4194304; -; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM60-NEXT: shr.u32 %r18, %r17, 16; -; SM60-NEXT: shl.b32 %r19, %r18, %r1; -; SM60-NEXT: and.b32 %r20, %r23, %r2; -; SM60-NEXT: or.b32 %r21, %r20, %r19; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM60-NEXT: mov.b32 %r23, %r3; -; SM60-NEXT: @%p2 bra $L__BB79_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r22, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r22; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<3>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11; +; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM60-NOFTZ-NEXT: @%p2 bra $L__BB79_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<3>; +; SM60-FTZ-NEXT: .reg .b16 %rs<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<24>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11; +; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-FTZ-NEXT: mov.b32 %r23, %r3; +; SM60-FTZ-NEXT: @%p2 bra $L__BB79_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fmin_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<3>; -; SM60-NEXT: .reg .b16 %rs<2>; -; SM60-NEXT: .reg .b32 %r<24>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: $L__BB80_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r23, %r1; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: min.f32 %r12, %r9, %r11; -; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM60-NEXT: add.s32 %r14, %r13, %r12; -; SM60-NEXT: add.s32 %r15, %r14, 32767; -; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM60-NEXT: or.b32 %r16, %r12, 4194304; -; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM60-NEXT: shr.u32 %r18, %r17, 16; -; SM60-NEXT: shl.b32 %r19, %r18, %r1; -; SM60-NEXT: and.b32 %r20, %r23, %r2; -; SM60-NEXT: or.b32 %r21, %r20, %r19; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM60-NEXT: mov.b32 %r23, %r3; -; SM60-NEXT: @%p2 bra $L__BB80_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r22, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r22; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<3>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: min.f32 %r12, %r9, %r11; +; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM60-NOFTZ-NEXT: @%p2 bra $L__BB80_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<3>; +; SM60-FTZ-NEXT: .reg .b16 %rs<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<24>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11; +; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-FTZ-NEXT: mov.b32 %r23, %r3; +; SM60-FTZ-NEXT: @%p2 bra $L__BB80_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fmax_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<3>; -; SM60-NEXT: .reg .b16 %rs<2>; -; SM60-NEXT: .reg .b32 %r<24>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r23, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: $L__BB81_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r23, %r1; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: max.f32 %r12, %r9, %r11; -; SM60-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM60-NEXT: add.s32 %r14, %r13, %r12; -; SM60-NEXT: add.s32 %r15, %r14, 32767; -; SM60-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM60-NEXT: or.b32 %r16, %r12, 4194304; -; SM60-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM60-NEXT: shr.u32 %r18, %r17, 16; -; SM60-NEXT: shl.b32 %r19, %r18, %r1; -; SM60-NEXT: and.b32 %r20, %r23, %r2; -; SM60-NEXT: or.b32 %r21, %r20, %r19; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM60-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM60-NEXT: mov.b32 %r23, %r3; -; SM60-NEXT: @%p2 bra $L__BB81_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r22, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r22; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<3>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: max.f32 %r12, %r9, %r11; +; SM60-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM60-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM60-NOFTZ-NEXT: @%p2 bra $L__BB81_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<3>; +; SM60-FTZ-NEXT: .reg .b16 %rs<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<24>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r23, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11; +; SM60-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM60-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM60-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM60-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM60-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM60-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM60-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM60-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM60-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM60-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM60-FTZ-NEXT: mov.b32 %r23, %r3; +; SM60-FTZ-NEXT: @%p2 bra $L__BB81_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fminimum_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<7>; -; SM60-NEXT: .reg .b16 %rs<8>; -; SM60-NEXT: .reg .b32 %r<20>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: setp.eq.b16 %p4, %rs1, -32768; -; SM60-NEXT: $L__BB82_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r19, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: setp.lt.f32 %p1, %r9, %r11; -; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11; -; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; -; SM60-NEXT: setp.eq.b16 %p3, %rs2, -32768; -; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM60-NEXT: cvt.u32.u16 %r12, %rs4; -; SM60-NEXT: shl.b32 %r13, %r12, 16; -; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; -; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; -; SM60-NEXT: cvt.u32.u16 %r14, %rs7; -; SM60-NEXT: shl.b32 %r15, %r14, %r1; -; SM60-NEXT: and.b32 %r16, %r19, %r2; -; SM60-NEXT: or.b32 %r17, %r16, %r15; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; -; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19; -; SM60-NEXT: mov.b32 %r19, %r3; -; SM60-NEXT: @%p6 bra $L__BB82_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r18, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r18; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<7>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<8>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<20>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM60-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: setp.lt.f32 %p1, %r9, %r11; +; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11; +; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM60-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; +; SM60-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM60-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM60-NOFTZ-NEXT: mov.b32 %r19, %r3; +; SM60-NOFTZ-NEXT: @%p6 bra $L__BB82_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<7>; +; SM60-FTZ-NEXT: .reg .b16 %rs<8>; +; SM60-FTZ-NEXT: .reg .b32 %r<20>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM60-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: setp.lt.ftz.f32 %p1, %r9, %r11; +; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11; +; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM60-FTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000; +; SM60-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM60-FTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM60-FTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM60-FTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM60-FTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM60-FTZ-NEXT: mov.b32 %r19, %r3; +; SM60-FTZ-NEXT: @%p6 bra $L__BB82_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM60-LABEL: fmaximum_acq_rel_bfloat_global_cta( -; SM60: { -; SM60-NEXT: .reg .pred %p<7>; -; SM60-NEXT: .reg .b16 %rs<8>; -; SM60-NEXT: .reg .b32 %r<20>; -; SM60-NEXT: .reg .b64 %rd<3>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; -; SM60-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; -; SM60-NEXT: membar.cta; -; SM60-NEXT: and.b64 %rd1, %rd2, -4; -; SM60-NEXT: cvt.u32.u64 %r4, %rd2; -; SM60-NEXT: and.b32 %r5, %r4, 3; -; SM60-NEXT: shl.b32 %r1, %r5, 3; -; SM60-NEXT: mov.b32 %r6, 65535; -; SM60-NEXT: shl.b32 %r7, %r6, %r1; -; SM60-NEXT: not.b32 %r2, %r7; -; SM60-NEXT: ld.volatile.global.b32 %r19, [%rd1]; -; SM60-NEXT: cvt.u32.u16 %r10, %rs1; -; SM60-NEXT: shl.b32 %r11, %r10, 16; -; SM60-NEXT: setp.eq.b16 %p4, %rs1, 0; -; SM60-NEXT: $L__BB83_1: // %atomicrmw.start -; SM60-NEXT: // =>This Inner Loop Header: Depth=1 -; SM60-NEXT: shr.u32 %r8, %r19, %r1; -; SM60-NEXT: cvt.u16.u32 %rs2, %r8; -; SM60-NEXT: shl.b32 %r9, %r8, 16; -; SM60-NEXT: setp.gt.f32 %p1, %r9, %r11; -; SM60-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM60-NEXT: setp.nan.f32 %p2, %r9, %r11; -; SM60-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; -; SM60-NEXT: setp.eq.b16 %p3, %rs2, 0; -; SM60-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM60-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM60-NEXT: cvt.u32.u16 %r12, %rs4; -; SM60-NEXT: shl.b32 %r13, %r12, 16; -; SM60-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; -; SM60-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; -; SM60-NEXT: cvt.u32.u16 %r14, %rs7; -; SM60-NEXT: shl.b32 %r15, %r14, %r1; -; SM60-NEXT: and.b32 %r16, %r19, %r2; -; SM60-NEXT: or.b32 %r17, %r16, %r15; -; SM60-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; -; SM60-NEXT: setp.ne.b32 %p6, %r3, %r19; -; SM60-NEXT: mov.b32 %r19, %r3; -; SM60-NEXT: @%p6 bra $L__BB83_1; -; SM60-NEXT: // %bb.2: // %atomicrmw.end -; SM60-NEXT: shr.u32 %r18, %r3, %r1; -; SM60-NEXT: membar.cta; -; SM60-NEXT: st.param.b16 [func_retval0], %r18; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<7>; +; SM60-NOFTZ-NEXT: .reg .b16 %rs<8>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<20>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM60-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1]; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM60-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM60-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-NOFTZ-NEXT: setp.gt.f32 %p1, %r9, %r11; +; SM60-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11; +; SM60-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM60-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM60-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM60-NOFTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM60-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; +; SM60-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM60-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM60-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM60-NOFTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM60-NOFTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM60-NOFTZ-NEXT: mov.b32 %r19, %r3; +; SM60-NOFTZ-NEXT: @%p6 bra $L__BB83_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<7>; +; SM60-FTZ-NEXT: .reg .b16 %rs<8>; +; SM60-FTZ-NEXT: .reg .b32 %r<20>; +; SM60-FTZ-NEXT: .reg .b64 %rd<3>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM60-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM60-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM60-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM60-FTZ-NEXT: mov.b32 %r6, 65535; +; SM60-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM60-FTZ-NEXT: not.b32 %r2, %r7; +; SM60-FTZ-NEXT: ld.volatile.global.b32 %r19, [%rd1]; +; SM60-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM60-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM60-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM60-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM60-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM60-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM60-FTZ-NEXT: setp.gt.ftz.f32 %p1, %r9, %r11; +; SM60-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM60-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11; +; SM60-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM60-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM60-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM60-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM60-FTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM60-FTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM60-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000; +; SM60-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM60-FTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM60-FTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM60-FTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM60-FTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM60-FTZ-NEXT: atom.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM60-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM60-FTZ-NEXT: mov.b32 %r19, %r3; +; SM60-FTZ-NEXT: @%p6 bra $L__BB83_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } @@ -3254,17 +3952,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_global_sys( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; -; SM60-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM60-NOFTZ-NEXT: membar.sys; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.sys.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB106_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.sys; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .b32 %r<3>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; +; SM60-FTZ-NEXT: atom.sys.global.add.f32 %r2, [%rd1], %r1; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel ret float %retval } @@ -3403,17 +4124,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_global_cluster( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; -; SM60-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB114_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .b32 %r<3>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; +; SM60-FTZ-NEXT: atom.cta.global.add.f32 %r2, [%rd1], %r1; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel ret float %retval } @@ -3552,17 +4296,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_global_gpu( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; -; SM60-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM60-NOFTZ-NEXT: membar.gl; +; SM60-NOFTZ-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB122_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.gl; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .b32 %r<3>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM60-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; +; SM60-FTZ-NEXT: atom.gpu.global.add.f32 %r2, [%rd1], %r1; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel ret float %retval } @@ -3701,17 +4468,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) { } define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_generic_cta( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1]; -; SM60-NEXT: atom.cta.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .pred %p<2>; +; SM60-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: ld.volatile.b32 %r4, [%rd1]; +; SM60-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM60-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM60-NOFTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM60-NOFTZ-NEXT: @%p1 bra $L__BB130_1; +; SM60-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-NOFTZ-NEXT: membar.cta; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<5>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.b32 %r4, [%rd1]; +; SM60-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM60-FTZ-NEXT: atom.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-FTZ-NEXT: mov.b32 %r4, %r1; +; SM60-FTZ-NEXT: @%p1 bra $L__BB130_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -3850,17 +4651,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) { } define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) { -; SM60-LABEL: fadd_acq_rel_float_shared_cta( -; SM60: { -; SM60-NEXT: .reg .b32 %r<3>; -; SM60-NEXT: .reg .b64 %rd<2>; -; SM60-EMPTY: -; SM60-NEXT: // %bb.0: -; SM60-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; -; SM60-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; -; SM60-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1; -; SM60-NEXT: st.param.b32 [func_retval0], %r2; -; SM60-NEXT: ret; +; SM60-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM60-NOFTZ: { +; SM60-NOFTZ-NEXT: .reg .b32 %r<3>; +; SM60-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM60-NOFTZ-EMPTY: +; SM60-NOFTZ-NEXT: // %bb.0: +; SM60-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM60-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; +; SM60-NOFTZ-NEXT: atom.cta.shared.add.f32 %r2, [%rd1], %r1; +; SM60-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM60-NOFTZ-NEXT: ret; +; +; SM60-FTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM60-FTZ: { +; SM60-FTZ-NEXT: .reg .pred %p<2>; +; SM60-FTZ-NEXT: .reg .b32 %r<5>; +; SM60-FTZ-NEXT: .reg .b64 %rd<2>; +; SM60-FTZ-EMPTY: +; SM60-FTZ-NEXT: // %bb.0: +; SM60-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1]; +; SM60-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: ld.volatile.shared.b32 %r4, [%rd1]; +; SM60-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start +; SM60-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM60-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM60-FTZ-NEXT: atom.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3; +; SM60-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM60-FTZ-NEXT: mov.b32 %r4, %r1; +; SM60-FTZ-NEXT: @%p1 bra $L__BB138_1; +; SM60-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM60-FTZ-NEXT: membar.cta; +; SM60-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM60-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel ret float %retval } diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll index b0fd2e6758d55b..10744421115b41 100644 --- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll +++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm70.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 -; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefix=SM70 +; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | FileCheck %s --check-prefixes=SM70,SM70-NOFTZ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM70,SM70-FTZ ; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 | %ptxas-verify -arch=sm_70 %} +; RUN: %if ptxas-sm_70 && ptxas-isa-6.3 %{ llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx63 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_70 %} define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM70-LABEL: xchg_acq_rel_i8_global_cta( @@ -1737,168 +1739,322 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB60_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .b32 %r<3>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fsub_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b32 %r<5>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM70-NEXT: $L__BB61_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: sub.rn.f32 %r3, %r4, %r2; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM70-NEXT: mov.b32 %r4, %r1; -; SM70-NEXT: @%p1 bra $L__BB61_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b32 [func_retval0], %r1; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB61_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<5>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-FTZ-NEXT: mov.b32 %r4, %r1; +; SM70-FTZ-NEXT: @%p1 bra $L__BB61_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fmin_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b32 %r<5>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM70-NEXT: $L__BB62_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: min.f32 %r3, %r4, %r2; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM70-NEXT: mov.b32 %r4, %r1; -; SM70-NEXT: @%p1 bra $L__BB62_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b32 [func_retval0], %r1; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: min.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB62_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<5>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-FTZ-NEXT: mov.b32 %r4, %r1; +; SM70-FTZ-NEXT: @%p1 bra $L__BB62_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fmax_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b32 %r<5>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM70-NEXT: $L__BB63_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: max.f32 %r3, %r4, %r2; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM70-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM70-NEXT: mov.b32 %r4, %r1; -; SM70-NEXT: @%p1 bra $L__BB63_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b32 [func_retval0], %r1; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: max.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB63_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<5>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-FTZ-NEXT: mov.b32 %r4, %r1; +; SM70-FTZ-NEXT: @%p1 bra $L__BB63_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fminimum_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<6>; -; SM70-NEXT: .reg .b32 %r<9>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; -; SM70-NEXT: setp.eq.b32 %p3, %r2, -2147483648; -; SM70-NEXT: $L__BB64_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: setp.nan.f32 %p1, %r8, %r2; -; SM70-NEXT: min.f32 %r3, %r8, %r2; -; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; -; SM70-NEXT: setp.eq.b32 %p2, %r8, -2147483648; -; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2; -; SM70-NEXT: selp.f32 %r6, %r2, %r5, %p3; -; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; -; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; -; SM70-NEXT: setp.ne.b32 %p5, %r1, %r8; -; SM70-NEXT: mov.b32 %r8, %r1; -; SM70-NEXT: @%p5 bra $L__BB64_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b32 [func_retval0], %r1; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<6>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<9>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; +; SM70-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648; +; SM70-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2; +; SM70-NOFTZ-NEXT: min.f32 %r3, %r8, %r2; +; SM70-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM70-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648; +; SM70-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM70-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; +; SM70-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM70-NOFTZ-NEXT: mov.b32 %r8, %r1; +; SM70-NOFTZ-NEXT: @%p5 bra $L__BB64_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<6>; +; SM70-FTZ-NEXT: .reg .b32 %r<9>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; +; SM70-FTZ-NEXT: setp.eq.b32 %p3, %r2, -2147483648; +; SM70-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2; +; SM70-FTZ-NEXT: min.ftz.f32 %r3, %r8, %r2; +; SM70-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM70-FTZ-NEXT: setp.eq.b32 %p2, %r8, -2147483648; +; SM70-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM70-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000; +; SM70-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM70-FTZ-NEXT: mov.b32 %r8, %r1; +; SM70-FTZ-NEXT: @%p5 bra $L__BB64_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fmaximum_acq_rel_float_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<6>; -; SM70-NEXT: .reg .b32 %r<9>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; -; SM70-NEXT: setp.eq.b32 %p3, %r2, 0; -; SM70-NEXT: $L__BB65_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: setp.nan.f32 %p1, %r8, %r2; -; SM70-NEXT: max.f32 %r3, %r8, %r2; -; SM70-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; -; SM70-NEXT: setp.eq.b32 %p2, %r8, 0; -; SM70-NEXT: selp.f32 %r5, %r8, %r4, %p2; -; SM70-NEXT: selp.f32 %r6, %r2, %r5, %p3; -; SM70-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; -; SM70-NEXT: selp.f32 %r7, %r6, %r4, %p4; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; -; SM70-NEXT: setp.ne.b32 %p5, %r1, %r8; -; SM70-NEXT: mov.b32 %r8, %r1; -; SM70-NEXT: @%p5 bra $L__BB65_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b32 [func_retval0], %r1; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<6>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<9>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; +; SM70-NOFTZ-NEXT: setp.eq.b32 %p3, %r2, 0; +; SM70-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r8, %r2; +; SM70-NOFTZ-NEXT: max.f32 %r3, %r8, %r2; +; SM70-NOFTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM70-NOFTZ-NEXT: setp.eq.b32 %p2, %r8, 0; +; SM70-NOFTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM70-NOFTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM70-NOFTZ-NEXT: setp.eq.f32 %p4, %r4, 0f00000000; +; SM70-NOFTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM70-NOFTZ-NEXT: mov.b32 %r8, %r1; +; SM70-NOFTZ-NEXT: @%p5 bra $L__BB65_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<6>; +; SM70-FTZ-NEXT: .reg .b32 %r<9>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r8, [%rd1]; +; SM70-FTZ-NEXT: setp.eq.b32 %p3, %r2, 0; +; SM70-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r8, %r2; +; SM70-FTZ-NEXT: max.ftz.f32 %r3, %r8, %r2; +; SM70-FTZ-NEXT: selp.f32 %r4, 0f7FC00000, %r3, %p1; +; SM70-FTZ-NEXT: setp.eq.b32 %p2, %r8, 0; +; SM70-FTZ-NEXT: selp.f32 %r5, %r8, %r4, %p2; +; SM70-FTZ-NEXT: selp.f32 %r6, %r2, %r5, %p3; +; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p4, %r4, 0f00000000; +; SM70-FTZ-NEXT: selp.f32 %r7, %r6, %r4, %p4; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r8, %r7; +; SM70-FTZ-NEXT: setp.ne.b32 %p5, %r1, %r8; +; SM70-FTZ-NEXT: mov.b32 %r8, %r1; +; SM70-FTZ-NEXT: @%p5 bra $L__BB65_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -2065,565 +2221,1107 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double } define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fadd_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .b16 %rs<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1; -; SM70-NEXT: st.param.b16 [func_retval0], %rs2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .b16 %rs<3>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b16 %rs<4>; +; SM70-FTZ-NEXT: .reg .b32 %r<15>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM70-FTZ-NEXT: mov.b32 %r14, %r3; +; SM70-FTZ-NEXT: @%p1 bra $L__BB72_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fsub_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b16 %rs<4>; -; SM70-NEXT: .reg .b32 %r<15>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM70-NEXT: $L__BB73_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r14, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; -; SM70-NEXT: cvt.u32.u16 %r9, %rs3; -; SM70-NEXT: shl.b32 %r10, %r9, %r1; -; SM70-NEXT: and.b32 %r11, %r14, %r2; -; SM70-NEXT: or.b32 %r12, %r11, %r10; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM70-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM70-NEXT: mov.b32 %r14, %r3; -; SM70-NEXT: @%p1 bra $L__BB73_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r13, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r13; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB73_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b16 %rs<4>; +; SM70-FTZ-NEXT: .reg .b32 %r<15>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM70-FTZ-NEXT: mov.b32 %r14, %r3; +; SM70-FTZ-NEXT: @%p1 bra $L__BB73_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fmin_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b16 %rs<4>; -; SM70-NEXT: .reg .b32 %r<18>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; -; SM70-NEXT: cvt.f32.f16 %r10, %rs1; -; SM70-NEXT: $L__BB74_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r17, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: cvt.f32.f16 %r9, %rs2; -; SM70-NEXT: min.f32 %r11, %r9, %r10; -; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11; -; SM70-NEXT: cvt.u32.u16 %r12, %rs3; -; SM70-NEXT: shl.b32 %r13, %r12, %r1; -; SM70-NEXT: and.b32 %r14, %r17, %r2; -; SM70-NEXT: or.b32 %r15, %r14, %r13; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; -; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17; -; SM70-NEXT: mov.b32 %r17, %r3; -; SM70-NEXT: @%p1 bra $L__BB74_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r16, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r16; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<18>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1; +; SM70-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2; +; SM70-NOFTZ-NEXT: min.f32 %r11, %r9, %r10; +; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB74_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b16 %rs<4>; +; SM70-FTZ-NEXT: .reg .b32 %r<18>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; +; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1; +; SM70-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2; +; SM70-FTZ-NEXT: min.ftz.f32 %r11, %r9, %r10; +; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM70-FTZ-NEXT: mov.b32 %r17, %r3; +; SM70-FTZ-NEXT: @%p1 bra $L__BB74_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fmax_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<2>; -; SM70-NEXT: .reg .b16 %rs<4>; -; SM70-NEXT: .reg .b32 %r<18>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; -; SM70-NEXT: cvt.f32.f16 %r10, %rs1; -; SM70-NEXT: $L__BB75_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r17, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: cvt.f32.f16 %r9, %rs2; -; SM70-NEXT: max.f32 %r11, %r9, %r10; -; SM70-NEXT: cvt.rn.f16.f32 %rs3, %r11; -; SM70-NEXT: cvt.u32.u16 %r12, %rs3; -; SM70-NEXT: shl.b32 %r13, %r12, %r1; -; SM70-NEXT: and.b32 %r14, %r17, %r2; -; SM70-NEXT: or.b32 %r15, %r14, %r13; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; -; SM70-NEXT: setp.ne.b32 %p1, %r3, %r17; -; SM70-NEXT: mov.b32 %r17, %r3; -; SM70-NEXT: @%p1 bra $L__BB75_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r16, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r16; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<18>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.f32.f16 %r10, %rs1; +; SM70-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: cvt.f32.f16 %r9, %rs2; +; SM70-NOFTZ-NEXT: max.f32 %r11, %r9, %r10; +; SM70-NOFTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM70-NOFTZ-NEXT: mov.b32 %r17, %r3; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB75_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b16 %rs<4>; +; SM70-FTZ-NEXT: .reg .b32 %r<18>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r17, [%rd1]; +; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r10, %rs1; +; SM70-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r17, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: cvt.ftz.f32.f16 %r9, %rs2; +; SM70-FTZ-NEXT: max.ftz.f32 %r11, %r9, %r10; +; SM70-FTZ-NEXT: cvt.rn.f16.f32 %rs3, %r11; +; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs3; +; SM70-FTZ-NEXT: shl.b32 %r13, %r12, %r1; +; SM70-FTZ-NEXT: and.b32 %r14, %r17, %r2; +; SM70-FTZ-NEXT: or.b32 %r15, %r14, %r13; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r17, %r15; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r17; +; SM70-FTZ-NEXT: mov.b32 %r17, %r3; +; SM70-FTZ-NEXT: @%p1 bra $L__BB75_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r16, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r16; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fminimum_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<7>; -; SM70-NEXT: .reg .b16 %rs<9>; -; SM70-NEXT: .reg .b32 %r<15>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768; -; SM70-NEXT: $L__BB76_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r14, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: setp.lt.f16 %p1, %rs2, %rs1; -; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1; -; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; -; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768; -; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM70-NEXT: mov.b16 %rs7, 0x0000; -; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7; -; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; -; SM70-NEXT: cvt.u32.u16 %r9, %rs8; -; SM70-NEXT: shl.b32 %r10, %r9, %r1; -; SM70-NEXT: and.b32 %r11, %r14, %r2; -; SM70-NEXT: or.b32 %r12, %r11, %r10; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14; -; SM70-NEXT: mov.b32 %r14, %r3; -; SM70-NEXT: @%p6 bra $L__BB76_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r13, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r13; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<7>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<9>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM70-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: setp.lt.f16 %p1, %rs2, %rs1; +; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1; +; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-NOFTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM70-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7; +; SM70-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM70-NOFTZ-NEXT: @%p6 bra $L__BB76_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<7>; +; SM70-FTZ-NEXT: .reg .b16 %rs<9>; +; SM70-FTZ-NEXT: .reg .b32 %r<15>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM70-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: setp.lt.ftz.f16 %p1, %rs2, %rs1; +; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1; +; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-FTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7; +; SM70-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM70-FTZ-NEXT: mov.b32 %r14, %r3; +; SM70-FTZ-NEXT: @%p6 bra $L__BB76_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM70-LABEL: fmaximum_acq_rel_half_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<7>; -; SM70-NEXT: .reg .b16 %rs<9>; -; SM70-NEXT: .reg .b32 %r<15>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0; -; SM70-NEXT: $L__BB77_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r14, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: setp.gt.f16 %p1, %rs2, %rs1; -; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM70-NEXT: setp.nan.f16 %p2, %rs2, %rs1; -; SM70-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; -; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0; -; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM70-NEXT: mov.b16 %rs7, 0x0000; -; SM70-NEXT: setp.eq.f16 %p5, %rs4, %rs7; -; SM70-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; -; SM70-NEXT: cvt.u32.u16 %r9, %rs8; -; SM70-NEXT: shl.b32 %r10, %r9, %r1; -; SM70-NEXT: and.b32 %r11, %r14, %r2; -; SM70-NEXT: or.b32 %r12, %r11, %r10; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM70-NEXT: setp.ne.b32 %p6, %r3, %r14; -; SM70-NEXT: mov.b32 %r14, %r3; -; SM70-NEXT: @%p6 bra $L__BB77_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r13, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r13; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<7>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<9>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM70-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: setp.gt.f16 %p1, %rs2, %rs1; +; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-NOFTZ-NEXT: setp.nan.f16 %p2, %rs2, %rs1; +; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-NOFTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM70-NOFTZ-NEXT: setp.eq.f16 %p5, %rs4, %rs7; +; SM70-NOFTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM70-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM70-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM70-NOFTZ-NEXT: @%p6 bra $L__BB77_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<7>; +; SM70-FTZ-NEXT: .reg .b16 %rs<9>; +; SM70-FTZ-NEXT: .reg .b32 %r<15>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM70-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: setp.gt.ftz.f16 %p1, %rs2, %rs1; +; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-FTZ-NEXT: setp.nan.ftz.f16 %p2, %rs2, %rs1; +; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7E00, %rs3, %p2; +; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-FTZ-NEXT: mov.b16 %rs7, 0x0000; +; SM70-FTZ-NEXT: setp.eq.ftz.f16 %p5, %rs4, %rs7; +; SM70-FTZ-NEXT: selp.b16 %rs8, %rs6, %rs4, %p5; +; SM70-FTZ-NEXT: cvt.u32.u16 %r9, %rs8; +; SM70-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM70-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM70-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r14; +; SM70-FTZ-NEXT: mov.b32 %r14, %r3; +; SM70-FTZ-NEXT: @%p6 bra $L__BB77_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define bfloat @fadd_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fadd_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<3>; -; SM70-NEXT: .reg .b16 %rs<2>; -; SM70-NEXT: .reg .b32 %r<24>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: $L__BB78_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r23, %r1; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: add.rn.f32 %r12, %r9, %r11; -; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM70-NEXT: add.s32 %r14, %r13, %r12; -; SM70-NEXT: add.s32 %r15, %r14, 32767; -; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM70-NEXT: or.b32 %r16, %r12, 4194304; -; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM70-NEXT: shr.u32 %r18, %r17, 16; -; SM70-NEXT: shl.b32 %r19, %r18, %r1; -; SM70-NEXT: and.b32 %r20, %r23, %r2; -; SM70-NEXT: or.b32 %r21, %r20, %r19; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM70-NEXT: mov.b32 %r23, %r3; -; SM70-NEXT: @%p2 bra $L__BB78_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r22, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r22; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<3>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: $L__BB78_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: add.rn.f32 %r12, %r9, %r11; +; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM70-NOFTZ-NEXT: @%p2 bra $L__BB78_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<3>; +; SM70-FTZ-NEXT: .reg .b16 %rs<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<24>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: $L__BB78_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: add.rn.ftz.f32 %r12, %r9, %r11; +; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-FTZ-NEXT: mov.b32 %r23, %r3; +; SM70-FTZ-NEXT: @%p2 bra $L__BB78_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fsub_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fsub_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<3>; -; SM70-NEXT: .reg .b16 %rs<2>; -; SM70-NEXT: .reg .b32 %r<24>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: $L__BB79_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r23, %r1; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: sub.rn.f32 %r12, %r9, %r11; -; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM70-NEXT: add.s32 %r14, %r13, %r12; -; SM70-NEXT: add.s32 %r15, %r14, 32767; -; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM70-NEXT: or.b32 %r16, %r12, 4194304; -; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM70-NEXT: shr.u32 %r18, %r17, 16; -; SM70-NEXT: shl.b32 %r19, %r18, %r1; -; SM70-NEXT: and.b32 %r20, %r23, %r2; -; SM70-NEXT: or.b32 %r21, %r20, %r19; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM70-NEXT: mov.b32 %r23, %r3; -; SM70-NEXT: @%p2 bra $L__BB79_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r22, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r22; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fsub_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<3>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: $L__BB79_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: sub.rn.f32 %r12, %r9, %r11; +; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM70-NOFTZ-NEXT: @%p2 bra $L__BB79_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fsub_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<3>; +; SM70-FTZ-NEXT: .reg .b16 %rs<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<24>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: $L__BB79_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: sub.rn.ftz.f32 %r12, %r9, %r11; +; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-FTZ-NEXT: mov.b32 %r23, %r3; +; SM70-FTZ-NEXT: @%p2 bra $L__BB79_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmin_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fmin_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<3>; -; SM70-NEXT: .reg .b16 %rs<2>; -; SM70-NEXT: .reg .b32 %r<24>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: $L__BB80_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r23, %r1; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: min.f32 %r12, %r9, %r11; -; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM70-NEXT: add.s32 %r14, %r13, %r12; -; SM70-NEXT: add.s32 %r15, %r14, 32767; -; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM70-NEXT: or.b32 %r16, %r12, 4194304; -; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM70-NEXT: shr.u32 %r18, %r17, 16; -; SM70-NEXT: shl.b32 %r19, %r18, %r1; -; SM70-NEXT: and.b32 %r20, %r23, %r2; -; SM70-NEXT: or.b32 %r21, %r20, %r19; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM70-NEXT: mov.b32 %r23, %r3; -; SM70-NEXT: @%p2 bra $L__BB80_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r22, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r22; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmin_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<3>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: $L__BB80_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: min.f32 %r12, %r9, %r11; +; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM70-NOFTZ-NEXT: @%p2 bra $L__BB80_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmin_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<3>; +; SM70-FTZ-NEXT: .reg .b16 %rs<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<24>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: $L__BB80_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: min.ftz.f32 %r12, %r9, %r11; +; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-FTZ-NEXT: mov.b32 %r23, %r3; +; SM70-FTZ-NEXT: @%p2 bra $L__BB80_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmax_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fmax_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<3>; -; SM70-NEXT: .reg .b16 %rs<2>; -; SM70-NEXT: .reg .b32 %r<24>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: $L__BB81_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r23, %r1; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: max.f32 %r12, %r9, %r11; -; SM70-NEXT: bfe.u32 %r13, %r12, 16, 1; -; SM70-NEXT: add.s32 %r14, %r13, %r12; -; SM70-NEXT: add.s32 %r15, %r14, 32767; -; SM70-NEXT: setp.nan.f32 %p1, %r12, %r12; -; SM70-NEXT: or.b32 %r16, %r12, 4194304; -; SM70-NEXT: selp.b32 %r17, %r16, %r15, %p1; -; SM70-NEXT: shr.u32 %r18, %r17, 16; -; SM70-NEXT: shl.b32 %r19, %r18, %r1; -; SM70-NEXT: and.b32 %r20, %r23, %r2; -; SM70-NEXT: or.b32 %r21, %r20, %r19; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; -; SM70-NEXT: setp.ne.b32 %p2, %r3, %r23; -; SM70-NEXT: mov.b32 %r23, %r3; -; SM70-NEXT: @%p2 bra $L__BB81_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r22, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r22; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmax_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<3>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<24>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: $L__BB81_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: max.f32 %r12, %r9, %r11; +; SM70-NOFTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-NOFTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-NOFTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p1, %r12, %r12; +; SM70-NOFTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-NOFTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-NOFTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-NOFTZ-NEXT: mov.b32 %r23, %r3; +; SM70-NOFTZ-NEXT: @%p2 bra $L__BB81_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmax_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<3>; +; SM70-FTZ-NEXT: .reg .b16 %rs<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<24>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r23, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: $L__BB81_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r23, %r1; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: max.ftz.f32 %r12, %r9, %r11; +; SM70-FTZ-NEXT: bfe.u32 %r13, %r12, 16, 1; +; SM70-FTZ-NEXT: add.s32 %r14, %r13, %r12; +; SM70-FTZ-NEXT: add.s32 %r15, %r14, 32767; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p1, %r12, %r12; +; SM70-FTZ-NEXT: or.b32 %r16, %r12, 4194304; +; SM70-FTZ-NEXT: selp.b32 %r17, %r16, %r15, %p1; +; SM70-FTZ-NEXT: shr.u32 %r18, %r17, 16; +; SM70-FTZ-NEXT: shl.b32 %r19, %r18, %r1; +; SM70-FTZ-NEXT: and.b32 %r20, %r23, %r2; +; SM70-FTZ-NEXT: or.b32 %r21, %r20, %r19; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r23, %r21; +; SM70-FTZ-NEXT: setp.ne.b32 %p2, %r3, %r23; +; SM70-FTZ-NEXT: mov.b32 %r23, %r3; +; SM70-FTZ-NEXT: @%p2 bra $L__BB81_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r22, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r22; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fminimum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fminimum_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<7>; -; SM70-NEXT: .reg .b16 %rs<8>; -; SM70-NEXT: .reg .b32 %r<20>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: setp.eq.b16 %p4, %rs1, -32768; -; SM70-NEXT: $L__BB82_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r19, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: setp.lt.f32 %p1, %r9, %r11; -; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11; -; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; -; SM70-NEXT: setp.eq.b16 %p3, %rs2, -32768; -; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM70-NEXT: cvt.u32.u16 %r12, %rs4; -; SM70-NEXT: shl.b32 %r13, %r12, 16; -; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; -; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; -; SM70-NEXT: cvt.u32.u16 %r14, %rs7; -; SM70-NEXT: shl.b32 %r15, %r14, %r1; -; SM70-NEXT: and.b32 %r16, %r19, %r2; -; SM70-NEXT: or.b32 %r17, %r16, %r15; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; -; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19; -; SM70-NEXT: mov.b32 %r19, %r3; -; SM70-NEXT: @%p6 bra $L__BB82_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r18, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r18; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fminimum_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<7>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<8>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<20>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM70-NOFTZ-NEXT: $L__BB82_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: setp.lt.f32 %p1, %r9, %r11; +; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11; +; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM70-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; +; SM70-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM70-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM70-NOFTZ-NEXT: mov.b32 %r19, %r3; +; SM70-NOFTZ-NEXT: @%p6 bra $L__BB82_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fminimum_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<7>; +; SM70-FTZ-NEXT: .reg .b16 %rs<8>; +; SM70-FTZ-NEXT: .reg .b32 %r<20>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, -32768; +; SM70-FTZ-NEXT: $L__BB82_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: setp.lt.ftz.f32 %p1, %r9, %r11; +; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11; +; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, -32768; +; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM70-FTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000; +; SM70-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM70-FTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM70-FTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM70-FTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM70-FTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM70-FTZ-NEXT: mov.b32 %r19, %r3; +; SM70-FTZ-NEXT: @%p6 bra $L__BB82_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } define bfloat @fmaximum_acq_rel_bfloat_global_cta(ptr addrspace(1) %addr, bfloat %val) { -; SM70-LABEL: fmaximum_acq_rel_bfloat_global_cta( -; SM70: { -; SM70-NEXT: .reg .pred %p<7>; -; SM70-NEXT: .reg .b16 %rs<8>; -; SM70-NEXT: .reg .b32 %r<20>; -; SM70-NEXT: .reg .b64 %rd<3>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; -; SM70-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: and.b64 %rd1, %rd2, -4; -; SM70-NEXT: cvt.u32.u64 %r4, %rd2; -; SM70-NEXT: and.b32 %r5, %r4, 3; -; SM70-NEXT: shl.b32 %r1, %r5, 3; -; SM70-NEXT: mov.b32 %r6, 65535; -; SM70-NEXT: shl.b32 %r7, %r6, %r1; -; SM70-NEXT: not.b32 %r2, %r7; -; SM70-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; -; SM70-NEXT: cvt.u32.u16 %r10, %rs1; -; SM70-NEXT: shl.b32 %r11, %r10, 16; -; SM70-NEXT: setp.eq.b16 %p4, %rs1, 0; -; SM70-NEXT: $L__BB83_1: // %atomicrmw.start -; SM70-NEXT: // =>This Inner Loop Header: Depth=1 -; SM70-NEXT: shr.u32 %r8, %r19, %r1; -; SM70-NEXT: cvt.u16.u32 %rs2, %r8; -; SM70-NEXT: shl.b32 %r9, %r8, 16; -; SM70-NEXT: setp.gt.f32 %p1, %r9, %r11; -; SM70-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; -; SM70-NEXT: setp.nan.f32 %p2, %r9, %r11; -; SM70-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; -; SM70-NEXT: setp.eq.b16 %p3, %rs2, 0; -; SM70-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; -; SM70-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; -; SM70-NEXT: cvt.u32.u16 %r12, %rs4; -; SM70-NEXT: shl.b32 %r13, %r12, 16; -; SM70-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; -; SM70-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; -; SM70-NEXT: cvt.u32.u16 %r14, %rs7; -; SM70-NEXT: shl.b32 %r15, %r14, %r1; -; SM70-NEXT: and.b32 %r16, %r19, %r2; -; SM70-NEXT: or.b32 %r17, %r16, %r15; -; SM70-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; -; SM70-NEXT: setp.ne.b32 %p6, %r3, %r19; -; SM70-NEXT: mov.b32 %r19, %r3; -; SM70-NEXT: @%p6 bra $L__BB83_1; -; SM70-NEXT: // %bb.2: // %atomicrmw.end -; SM70-NEXT: shr.u32 %r18, %r3, %r1; -; SM70-NEXT: fence.acq_rel.cta; -; SM70-NEXT: st.param.b16 [func_retval0], %r18; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<7>; +; SM70-NOFTZ-NEXT: .reg .b16 %rs<8>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<20>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM70-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-NOFTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM70-NOFTZ-NEXT: $L__BB83_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM70-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-NOFTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-NOFTZ-NEXT: setp.gt.f32 %p1, %r9, %r11; +; SM70-NOFTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-NOFTZ-NEXT: setp.nan.f32 %p2, %r9, %r11; +; SM70-NOFTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM70-NOFTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM70-NOFTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-NOFTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM70-NOFTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM70-NOFTZ-NEXT: setp.eq.f32 %p5, %r13, 0f00000000; +; SM70-NOFTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM70-NOFTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM70-NOFTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM70-NOFTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM70-NOFTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM70-NOFTZ-NEXT: mov.b32 %r19, %r3; +; SM70-NOFTZ-NEXT: @%p6 bra $L__BB83_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fmaximum_acq_rel_bfloat_global_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<7>; +; SM70-FTZ-NEXT: .reg .b16 %rs<8>; +; SM70-FTZ-NEXT: .reg .b32 %r<20>; +; SM70-FTZ-NEXT: .reg .b64 %rd<3>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_bfloat_global_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_bfloat_global_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM70-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM70-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM70-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM70-FTZ-NEXT: mov.b32 %r6, 65535; +; SM70-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM70-FTZ-NEXT: not.b32 %r2, %r7; +; SM70-FTZ-NEXT: ld.relaxed.cta.global.b32 %r19, [%rd1]; +; SM70-FTZ-NEXT: cvt.u32.u16 %r10, %rs1; +; SM70-FTZ-NEXT: shl.b32 %r11, %r10, 16; +; SM70-FTZ-NEXT: setp.eq.b16 %p4, %rs1, 0; +; SM70-FTZ-NEXT: $L__BB83_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: shr.u32 %r8, %r19, %r1; +; SM70-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM70-FTZ-NEXT: shl.b32 %r9, %r8, 16; +; SM70-FTZ-NEXT: setp.gt.ftz.f32 %p1, %r9, %r11; +; SM70-FTZ-NEXT: selp.b16 %rs3, %rs2, %rs1, %p1; +; SM70-FTZ-NEXT: setp.nan.ftz.f32 %p2, %r9, %r11; +; SM70-FTZ-NEXT: selp.b16 %rs4, 0x7FC0, %rs3, %p2; +; SM70-FTZ-NEXT: setp.eq.b16 %p3, %rs2, 0; +; SM70-FTZ-NEXT: selp.b16 %rs5, %rs2, %rs4, %p3; +; SM70-FTZ-NEXT: selp.b16 %rs6, %rs1, %rs5, %p4; +; SM70-FTZ-NEXT: cvt.u32.u16 %r12, %rs4; +; SM70-FTZ-NEXT: shl.b32 %r13, %r12, 16; +; SM70-FTZ-NEXT: setp.eq.ftz.f32 %p5, %r13, 0f00000000; +; SM70-FTZ-NEXT: selp.b16 %rs7, %rs6, %rs4, %p5; +; SM70-FTZ-NEXT: cvt.u32.u16 %r14, %rs7; +; SM70-FTZ-NEXT: shl.b32 %r15, %r14, %r1; +; SM70-FTZ-NEXT: and.b32 %r16, %r19, %r2; +; SM70-FTZ-NEXT: or.b32 %r17, %r16, %r15; +; SM70-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r19, %r17; +; SM70-FTZ-NEXT: setp.ne.b32 %p6, %r3, %r19; +; SM70-FTZ-NEXT: mov.b32 %r19, %r3; +; SM70-FTZ-NEXT: @%p6 bra $L__BB83_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: shr.u32 %r18, %r3, %r1; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b16 [func_retval0], %r18; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, bfloat %val syncscope("block") acq_rel ret bfloat %retval } @@ -3228,17 +3926,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_global_sys( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; -; SM70-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.sys; +; SM70-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB106_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.sys; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .b32 %r<3>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; +; SM70-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel ret float %retval } @@ -3377,17 +4098,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_global_cluster( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; -; SM70-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB114_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .b32 %r<3>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; +; SM70-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel ret float %retval } @@ -3526,17 +4270,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_global_gpu( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; -; SM70-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.gpu; +; SM70-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB122_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.gpu; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .b32 %r<3>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM70-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; +; SM70-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel ret float %retval } @@ -3675,17 +4442,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) { } define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_generic_cta( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1]; -; SM70-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .pred %p<2>; +; SM70-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; +; SM70-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM70-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM70-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM70-NOFTZ-NEXT: @%p1 bra $L__BB130_1; +; SM70-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-NOFTZ-NEXT: fence.acq_rel.cta; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<5>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; +; SM70-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM70-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-FTZ-NEXT: mov.b32 %r4, %r1; +; SM70-FTZ-NEXT: @%p1 bra $L__BB130_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -3824,17 +4625,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) { } define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) { -; SM70-LABEL: fadd_acq_rel_float_shared_cta( -; SM70: { -; SM70-NEXT: .reg .b32 %r<3>; -; SM70-NEXT: .reg .b64 %rd<2>; -; SM70-EMPTY: -; SM70-NEXT: // %bb.0: -; SM70-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; -; SM70-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; -; SM70-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1; -; SM70-NEXT: st.param.b32 [func_retval0], %r2; -; SM70-NEXT: ret; +; SM70-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM70-NOFTZ: { +; SM70-NOFTZ-NEXT: .reg .b32 %r<3>; +; SM70-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM70-NOFTZ-EMPTY: +; SM70-NOFTZ-NEXT: // %bb.0: +; SM70-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM70-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; +; SM70-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1; +; SM70-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM70-NOFTZ-NEXT: ret; +; +; SM70-FTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM70-FTZ: { +; SM70-FTZ-NEXT: .reg .pred %p<2>; +; SM70-FTZ-NEXT: .reg .b32 %r<5>; +; SM70-FTZ-NEXT: .reg .b64 %rd<2>; +; SM70-FTZ-EMPTY: +; SM70-FTZ-NEXT: // %bb.0: +; SM70-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1]; +; SM70-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1]; +; SM70-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start +; SM70-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM70-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM70-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3; +; SM70-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM70-FTZ-NEXT: mov.b32 %r4, %r1; +; SM70-FTZ-NEXT: @%p1 bra $L__BB138_1; +; SM70-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM70-FTZ-NEXT: fence.acq_rel.cta; +; SM70-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM70-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel ret float %retval } diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll index e072e921f80ea5..46819ca53638ce 100644 --- a/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll +++ b/llvm/test/CodeGen/NVPTX/atomicrmw-sm90.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 -; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefix=SM90 +; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | FileCheck %s --check-prefixes=SM90,SM90-NOFTZ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM90,SM90-FTZ ; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 | %ptxas-verify -arch=sm_90 %} +; RUN: %if ptxas-sm_90 && ptxas-isa-8.7 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx87 -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_90 %} define i8 @xchg_acq_rel_i8_global_cta(ptr addrspace(1) %addr, i8 %val) { ; SM90-LABEL: xchg_acq_rel_i8_global_cta( @@ -1737,152 +1739,290 @@ define i64 @usub_sat_acq_rel_i64_global_cta(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB60_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB60_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .b32 %r<3>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: atom.acq_rel.cta.global.add.f32 %r2, [%rd1], %r1; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fsub_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fsub_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b32 %r<5>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM90-NEXT: $L__BB61_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: sub.rn.f32 %r3, %r4, %r2; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM90-NEXT: mov.b32 %r4, %r1; -; SM90-NEXT: @%p1 bra $L__BB61_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b32 [func_retval0], %r1; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: sub.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB61_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fsub_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fsub_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fsub_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB61_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: sub.rn.ftz.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB61_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmin_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fmin_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b32 %r<5>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM90-NEXT: $L__BB62_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: min.f32 %r3, %r4, %r2; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM90-NEXT: mov.b32 %r4, %r1; -; SM90-NEXT: @%p1 bra $L__BB62_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b32 [func_retval0], %r1; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: min.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB62_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmin_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmin_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmin_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB62_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: min.ftz.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB62_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmax_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fmax_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b32 %r<5>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM90-NEXT: $L__BB63_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: max.f32 %r3, %r4, %r2; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM90-NEXT: mov.b32 %r4, %r1; -; SM90-NEXT: @%p1 bra $L__BB63_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b32 [func_retval0], %r1; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: max.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB63_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmax_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmax_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmax_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB63_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: max.ftz.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB63_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fminimum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fminimum_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b32 %r<5>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM90-NEXT: $L__BB64_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: min.NaN.f32 %r3, %r4, %r2; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM90-NEXT: mov.b32 %r4, %r1; -; SM90-NEXT: @%p1 bra $L__BB64_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b32 [func_retval0], %r1; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: min.NaN.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB64_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fminimum_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fminimum_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fminimum_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB64_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: min.ftz.NaN.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB64_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } define float @fmaximum_acq_rel_float_global_cta(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fmaximum_acq_rel_float_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b32 %r<5>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; -; SM90-NEXT: $L__BB65_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: max.NaN.f32 %r3, %r4, %r2; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; -; SM90-NEXT: setp.ne.b32 %p1, %r1, %r4; -; SM90-NEXT: mov.b32 %r4, %r1; -; SM90-NEXT: @%p1 bra $L__BB65_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b32 [func_retval0], %r1; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: max.NaN.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB65_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmaximum_acq_rel_float_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fmaximum_acq_rel_float_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fmaximum_acq_rel_float_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB65_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: max.ftz.NaN.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB65_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -2049,227 +2189,455 @@ define double @fmaximum_acq_rel_double_global_cta(ptr addrspace(1) %addr, double } define half @fadd_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fadd_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .b16 %rs<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1; -; SM90-NEXT: st.param.b16 [func_retval0], %rs2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .b16 %rs<3>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: atom.acq_rel.cta.global.add.noftz.f16 %rs2, [%rd1], %rs1; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %rs2; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fadd_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fadd_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB72_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: add.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB72_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fsub_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fsub_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b16 %rs<4>; -; SM90-NEXT: .reg .b32 %r<15>; -; SM90-NEXT: .reg .b64 %rd<3>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: and.b64 %rd1, %rd2, -4; -; SM90-NEXT: cvt.u32.u64 %r4, %rd2; -; SM90-NEXT: and.b32 %r5, %r4, 3; -; SM90-NEXT: shl.b32 %r1, %r5, 3; -; SM90-NEXT: mov.b32 %r6, 65535; -; SM90-NEXT: shl.b32 %r7, %r6, %r1; -; SM90-NEXT: not.b32 %r2, %r7; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM90-NEXT: $L__BB73_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: shr.u32 %r8, %r14, %r1; -; SM90-NEXT: cvt.u16.u32 %rs2, %r8; -; SM90-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; -; SM90-NEXT: cvt.u32.u16 %r9, %rs3; -; SM90-NEXT: shl.b32 %r10, %r9, %r1; -; SM90-NEXT: and.b32 %r11, %r14, %r2; -; SM90-NEXT: or.b32 %r12, %r11, %r10; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM90-NEXT: mov.b32 %r14, %r3; -; SM90-NEXT: @%p1 bra $L__BB73_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: shr.u32 %r13, %r3, %r1; -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b16 [func_retval0], %r13; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-NOFTZ-NEXT: sub.rn.f16 %rs3, %rs2, %rs1; +; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB73_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fsub_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fsub_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fsub_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB73_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: sub.rn.ftz.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB73_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fsub ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmin_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fmin_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b16 %rs<4>; -; SM90-NEXT: .reg .b32 %r<15>; -; SM90-NEXT: .reg .b64 %rd<3>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: and.b64 %rd1, %rd2, -4; -; SM90-NEXT: cvt.u32.u64 %r4, %rd2; -; SM90-NEXT: and.b32 %r5, %r4, 3; -; SM90-NEXT: shl.b32 %r1, %r5, 3; -; SM90-NEXT: mov.b32 %r6, 65535; -; SM90-NEXT: shl.b32 %r7, %r6, %r1; -; SM90-NEXT: not.b32 %r2, %r7; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM90-NEXT: $L__BB74_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: shr.u32 %r8, %r14, %r1; -; SM90-NEXT: cvt.u16.u32 %rs2, %r8; -; SM90-NEXT: min.f16 %rs3, %rs2, %rs1; -; SM90-NEXT: cvt.u32.u16 %r9, %rs3; -; SM90-NEXT: shl.b32 %r10, %r9, %r1; -; SM90-NEXT: and.b32 %r11, %r14, %r2; -; SM90-NEXT: or.b32 %r12, %r11, %r10; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM90-NEXT: mov.b32 %r14, %r3; -; SM90-NEXT: @%p1 bra $L__BB74_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: shr.u32 %r13, %r3, %r1; -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b16 [func_retval0], %r13; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-NOFTZ-NEXT: min.f16 %rs3, %rs2, %rs1; +; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB74_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmin_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmin_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmin_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB74_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: min.ftz.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB74_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmin ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmax_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fmax_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b16 %rs<4>; -; SM90-NEXT: .reg .b32 %r<15>; -; SM90-NEXT: .reg .b64 %rd<3>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: and.b64 %rd1, %rd2, -4; -; SM90-NEXT: cvt.u32.u64 %r4, %rd2; -; SM90-NEXT: and.b32 %r5, %r4, 3; -; SM90-NEXT: shl.b32 %r1, %r5, 3; -; SM90-NEXT: mov.b32 %r6, 65535; -; SM90-NEXT: shl.b32 %r7, %r6, %r1; -; SM90-NEXT: not.b32 %r2, %r7; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM90-NEXT: $L__BB75_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: shr.u32 %r8, %r14, %r1; -; SM90-NEXT: cvt.u16.u32 %rs2, %r8; -; SM90-NEXT: max.f16 %rs3, %rs2, %rs1; -; SM90-NEXT: cvt.u32.u16 %r9, %rs3; -; SM90-NEXT: shl.b32 %r10, %r9, %r1; -; SM90-NEXT: and.b32 %r11, %r14, %r2; -; SM90-NEXT: or.b32 %r12, %r11, %r10; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM90-NEXT: mov.b32 %r14, %r3; -; SM90-NEXT: @%p1 bra $L__BB75_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: shr.u32 %r13, %r3, %r1; -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b16 [func_retval0], %r13; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-NOFTZ-NEXT: max.f16 %rs3, %rs2, %rs1; +; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB75_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmax_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmax_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmax_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB75_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: max.ftz.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB75_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmax ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fminimum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fminimum_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b16 %rs<4>; -; SM90-NEXT: .reg .b32 %r<15>; -; SM90-NEXT: .reg .b64 %rd<3>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: and.b64 %rd1, %rd2, -4; -; SM90-NEXT: cvt.u32.u64 %r4, %rd2; -; SM90-NEXT: and.b32 %r5, %r4, 3; -; SM90-NEXT: shl.b32 %r1, %r5, 3; -; SM90-NEXT: mov.b32 %r6, 65535; -; SM90-NEXT: shl.b32 %r7, %r6, %r1; -; SM90-NEXT: not.b32 %r2, %r7; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM90-NEXT: $L__BB76_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: shr.u32 %r8, %r14, %r1; -; SM90-NEXT: cvt.u16.u32 %rs2, %r8; -; SM90-NEXT: min.NaN.f16 %rs3, %rs2, %rs1; -; SM90-NEXT: cvt.u32.u16 %r9, %rs3; -; SM90-NEXT: shl.b32 %r10, %r9, %r1; -; SM90-NEXT: and.b32 %r11, %r14, %r2; -; SM90-NEXT: or.b32 %r12, %r11, %r10; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM90-NEXT: mov.b32 %r14, %r3; -; SM90-NEXT: @%p1 bra $L__BB76_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: shr.u32 %r13, %r3, %r1; -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b16 [func_retval0], %r13; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-NOFTZ-NEXT: min.NaN.f16 %rs3, %rs2, %rs1; +; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB76_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fminimum_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fminimum_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fminimum_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB76_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: min.ftz.NaN.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB76_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fminimum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } define half @fmaximum_acq_rel_half_global_cta(ptr addrspace(1) %addr, half %val) { -; SM90-LABEL: fmaximum_acq_rel_half_global_cta( -; SM90: { -; SM90-NEXT: .reg .pred %p<2>; -; SM90-NEXT: .reg .b16 %rs<4>; -; SM90-NEXT: .reg .b32 %r<15>; -; SM90-NEXT: .reg .b64 %rd<3>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; -; SM90-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; -; SM90-NEXT: fence.release.cta; -; SM90-NEXT: and.b64 %rd1, %rd2, -4; -; SM90-NEXT: cvt.u32.u64 %r4, %rd2; -; SM90-NEXT: and.b32 %r5, %r4, 3; -; SM90-NEXT: shl.b32 %r1, %r5, 3; -; SM90-NEXT: mov.b32 %r6, 65535; -; SM90-NEXT: shl.b32 %r7, %r6, %r1; -; SM90-NEXT: not.b32 %r2, %r7; -; SM90-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; -; SM90-NEXT: $L__BB77_1: // %atomicrmw.start -; SM90-NEXT: // =>This Inner Loop Header: Depth=1 -; SM90-NEXT: shr.u32 %r8, %r14, %r1; -; SM90-NEXT: cvt.u16.u32 %rs2, %r8; -; SM90-NEXT: max.NaN.f16 %rs3, %rs2, %rs1; -; SM90-NEXT: cvt.u32.u16 %r9, %rs3; -; SM90-NEXT: shl.b32 %r10, %r9, %r1; -; SM90-NEXT: and.b32 %r11, %r14, %r2; -; SM90-NEXT: or.b32 %r12, %r11, %r10; -; SM90-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; -; SM90-NEXT: setp.ne.b32 %p1, %r3, %r14; -; SM90-NEXT: mov.b32 %r14, %r3; -; SM90-NEXT: @%p1 bra $L__BB77_1; -; SM90-NEXT: // %bb.2: // %atomicrmw.end -; SM90-NEXT: shr.u32 %r13, %r3, %r1; -; SM90-NEXT: fence.acquire.cta; -; SM90-NEXT: st.param.b16 [func_retval0], %r13; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b16 %rs<4>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<15>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<3>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-NOFTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-NOFTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-NOFTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-NOFTZ-NEXT: mov.b32 %r6, 65535; +; SM90-NOFTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-NOFTZ-NEXT: not.b32 %r2, %r7; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-NOFTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-NOFTZ-NEXT: max.NaN.f16 %rs3, %rs2, %rs1; +; SM90-NOFTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-NOFTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-NOFTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-NOFTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-NOFTZ-NEXT: mov.b32 %r14, %r3; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB77_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fmaximum_acq_rel_half_global_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b16 %rs<4>; +; SM90-FTZ-NEXT: .reg .b32 %r<15>; +; SM90-FTZ-NEXT: .reg .b64 %rd<3>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b16 %rs1, [fmaximum_acq_rel_half_global_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd2, [fmaximum_acq_rel_half_global_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: and.b64 %rd1, %rd2, -4; +; SM90-FTZ-NEXT: cvt.u32.u64 %r4, %rd2; +; SM90-FTZ-NEXT: and.b32 %r5, %r4, 3; +; SM90-FTZ-NEXT: shl.b32 %r1, %r5, 3; +; SM90-FTZ-NEXT: mov.b32 %r6, 65535; +; SM90-FTZ-NEXT: shl.b32 %r7, %r6, %r1; +; SM90-FTZ-NEXT: not.b32 %r2, %r7; +; SM90-FTZ-NEXT: ld.relaxed.cta.global.b32 %r14, [%rd1]; +; SM90-FTZ-NEXT: $L__BB77_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: shr.u32 %r8, %r14, %r1; +; SM90-FTZ-NEXT: cvt.u16.u32 %rs2, %r8; +; SM90-FTZ-NEXT: max.ftz.NaN.f16 %rs3, %rs2, %rs1; +; SM90-FTZ-NEXT: cvt.u32.u16 %r9, %rs3; +; SM90-FTZ-NEXT: shl.b32 %r10, %r9, %r1; +; SM90-FTZ-NEXT: and.b32 %r11, %r14, %r2; +; SM90-FTZ-NEXT: or.b32 %r12, %r11, %r10; +; SM90-FTZ-NEXT: atom.relaxed.cta.global.cas.b32 %r3, [%rd1], %r14, %r12; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r3, %r14; +; SM90-FTZ-NEXT: mov.b32 %r14, %r3; +; SM90-FTZ-NEXT: @%p1 bra $L__BB77_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: shr.u32 %r13, %r3, %r1; +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b16 [func_retval0], %r13; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fmaximum ptr addrspace(1) %addr, half %val syncscope("block") acq_rel ret half %retval } @@ -3100,17 +3468,40 @@ define i64 @nand_acq_rel_i64_global_sys(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_sys(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_global_sys( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; -; SM90-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_sys_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM90-NOFTZ-NEXT: fence.release.sys; +; SM90-NOFTZ-NEXT: ld.relaxed.sys.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB106_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.sys.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB106_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.sys; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_global_sys( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .b32 %r<3>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_sys_param_0]; +; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_sys_param_1]; +; SM90-FTZ-NEXT: atom.acq_rel.sys.global.add.f32 %r2, [%rd1], %r1; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("") acq_rel ret float %retval } @@ -3249,17 +3640,40 @@ define i64 @nand_acq_rel_i64_global_cluster(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_cluster(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_global_cluster( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; -; SM90-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_cluster_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cluster; +; SM90-NOFTZ-NEXT: ld.relaxed.cluster.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB114_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cluster.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB114_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cluster; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_global_cluster( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .b32 %r<3>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_cluster_param_0]; +; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_cluster_param_1]; +; SM90-FTZ-NEXT: atom.acq_rel.cluster.global.add.f32 %r2, [%rd1], %r1; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("cluster") acq_rel ret float %retval } @@ -3398,17 +3812,40 @@ define i64 @nand_acq_rel_i64_global_gpu(ptr addrspace(1) %addr, i64 %val) { } define float @fadd_acq_rel_float_global_gpu(ptr addrspace(1) %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_global_gpu( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; -; SM90-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_global_gpu_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM90-NOFTZ-NEXT: fence.release.gpu; +; SM90-NOFTZ-NEXT: ld.relaxed.gpu.global.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB122_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.gpu.global.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB122_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.gpu; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_global_gpu( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .b32 %r<3>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_global_gpu_param_0]; +; SM90-FTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_global_gpu_param_1]; +; SM90-FTZ-NEXT: atom.acq_rel.gpu.global.add.f32 %r2, [%rd1], %r1; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(1) %addr, float %val syncscope("device") acq_rel ret float %retval } @@ -3547,17 +3984,51 @@ define i64 @nand_acq_rel_i64_generic_cta(ptr %addr, i64 %val) { } define float @fadd_acq_rel_float_generic_cta(ptr %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_generic_cta( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_generic_cta_param_1]; -; SM90-NEXT: atom.acq_rel.cta.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .pred %p<2>; +; SM90-NOFTZ-NEXT: .reg .b32 %r<5>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM90-NOFTZ-NEXT: fence.release.cta; +; SM90-NOFTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; +; SM90-NOFTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM90-NOFTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-NOFTZ-NEXT: add.rn.f32 %r3, %r4, %r2; +; SM90-NOFTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-NOFTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-NOFTZ-NEXT: mov.b32 %r4, %r1; +; SM90-NOFTZ-NEXT: @%p1 bra $L__BB130_1; +; SM90-NOFTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-NOFTZ-NEXT: fence.acquire.cta; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_generic_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_generic_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_generic_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB130_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB130_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr %addr, float %val syncscope("block") acq_rel ret float %retval } @@ -3696,17 +4167,40 @@ define i64 @nand_acq_rel_i64_shared_cta(ptr addrspace(3) %addr, i64 %val) { } define float @fadd_acq_rel_float_shared_cta(ptr addrspace(3) %addr, float %val) { -; SM90-LABEL: fadd_acq_rel_float_shared_cta( -; SM90: { -; SM90-NEXT: .reg .b32 %r<3>; -; SM90-NEXT: .reg .b64 %rd<2>; -; SM90-EMPTY: -; SM90-NEXT: // %bb.0: -; SM90-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; -; SM90-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; -; SM90-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1; -; SM90-NEXT: st.param.b32 [func_retval0], %r2; -; SM90-NEXT: ret; +; SM90-NOFTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM90-NOFTZ: { +; SM90-NOFTZ-NEXT: .reg .b32 %r<3>; +; SM90-NOFTZ-NEXT: .reg .b64 %rd<2>; +; SM90-NOFTZ-EMPTY: +; SM90-NOFTZ-NEXT: // %bb.0: +; SM90-NOFTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM90-NOFTZ-NEXT: ld.param.b32 %r1, [fadd_acq_rel_float_shared_cta_param_1]; +; SM90-NOFTZ-NEXT: atom.acq_rel.cta.shared.add.f32 %r2, [%rd1], %r1; +; SM90-NOFTZ-NEXT: st.param.b32 [func_retval0], %r2; +; SM90-NOFTZ-NEXT: ret; +; +; SM90-FTZ-LABEL: fadd_acq_rel_float_shared_cta( +; SM90-FTZ: { +; SM90-FTZ-NEXT: .reg .pred %p<2>; +; SM90-FTZ-NEXT: .reg .b32 %r<5>; +; SM90-FTZ-NEXT: .reg .b64 %rd<2>; +; SM90-FTZ-EMPTY: +; SM90-FTZ-NEXT: // %bb.0: +; SM90-FTZ-NEXT: ld.param.b32 %r2, [fadd_acq_rel_float_shared_cta_param_1]; +; SM90-FTZ-NEXT: ld.param.b64 %rd1, [fadd_acq_rel_float_shared_cta_param_0]; +; SM90-FTZ-NEXT: fence.release.cta; +; SM90-FTZ-NEXT: ld.relaxed.cta.shared.b32 %r4, [%rd1]; +; SM90-FTZ-NEXT: $L__BB138_1: // %atomicrmw.start +; SM90-FTZ-NEXT: // =>This Inner Loop Header: Depth=1 +; SM90-FTZ-NEXT: add.rn.ftz.f32 %r3, %r4, %r2; +; SM90-FTZ-NEXT: atom.relaxed.cta.shared.cas.b32 %r1, [%rd1], %r4, %r3; +; SM90-FTZ-NEXT: setp.ne.b32 %p1, %r1, %r4; +; SM90-FTZ-NEXT: mov.b32 %r4, %r1; +; SM90-FTZ-NEXT: @%p1 bra $L__BB138_1; +; SM90-FTZ-NEXT: // %bb.2: // %atomicrmw.end +; SM90-FTZ-NEXT: fence.acquire.cta; +; SM90-FTZ-NEXT: st.param.b32 [func_retval0], %r1; +; SM90-FTZ-NEXT: ret; %retval = atomicrmw fadd ptr addrspace(3) %addr, float %val syncscope("block") acq_rel ret float %retval } diff --git a/llvm/test/CodeGen/NVPTX/atomicrmw.py b/llvm/test/CodeGen/NVPTX/atomicrmw.py index ef1ce116f7cd04..966d62bf1b03af 100644 --- a/llvm/test/CodeGen/NVPTX/atomicrmw.py +++ b/llvm/test/CodeGen/NVPTX/atomicrmw.py @@ -54,9 +54,16 @@ """ ) +# atomicrmw fadd's lowering depends on the function's FTZ (denormal) mode, so we +# check codegen both with and without it. Lines common to both runs collapse to +# the SM${sm} prefix; only the FTZ-sensitive ops diverge into SM${sm}-NOFTZ / +# SM${sm}-FTZ. (-nvptx-allow-ftz-atomics is covered separately in +# atomicrmw-allow-ftz-atomics.ll.) run_statement = Template( - """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefix=SM${sm} + """; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | FileCheck %s --check-prefixes=SM${sm},SM${sm}-NOFTZ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | FileCheck %s --check-prefixes=SM${sm},SM${sm}-FTZ ; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} | %ptxas-verify -arch=sm_${sm} %} +; RUN: %if ptxas-sm_${sm} && ptxas-isa-${ptxfp} %{ llc < %s -march=nvptx64 -mcpu=sm_${sm} -mattr=+ptx${ptx} -denormal-fp-math-f32=preserve-sign | %ptxas-verify -arch=sm_${sm} %} """ ) diff --git a/llvm/test/CodeGen/NVPTX/atomics.ll b/llvm/test/CodeGen/NVPTX/atomics.ll index 3820b13f7da1f7..ab05bf1b7ea3f8 100644 --- a/llvm/test/CodeGen/NVPTX/atomics.ll +++ b/llvm/test/CodeGen/NVPTX/atomics.ll @@ -8,14 +8,23 @@ declare float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val) define float @atomic_add_f32_generic(ptr %addr, float %val) { ; CHECK-LABEL: atomic_add_f32_generic( ; CHECK: { -; CHECK-NEXT: .reg .b32 %r<3>; +; CHECK-NEXT: .reg .pred %p<2>; +; CHECK-NEXT: .reg .b32 %r<5>; ; CHECK-NEXT: .reg .b64 %rd<2>; ; CHECK-EMPTY: ; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_generic_param_1]; ; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_generic_param_0]; -; CHECK-NEXT: ld.param.b32 %r1, [atomic_add_f32_generic_param_1]; -; CHECK-NEXT: atom.add.f32 %r2, [%rd1], %r1; -; CHECK-NEXT: st.param.b32 [func_retval0], %r2; +; CHECK-NEXT: ld.volatile.b32 %r4, [%rd1]; +; CHECK-NEXT: $L__BB0_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2; +; CHECK-NEXT: atom.cas.b32 %r1, [%rd1], %r4, %r3; +; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4; +; CHECK-NEXT: mov.b32 %r4, %r1; +; CHECK-NEXT: @%p1 bra $L__BB0_1; +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: st.param.b32 [func_retval0], %r1; ; CHECK-NEXT: ret; %ret = call float @llvm.nvvm.atomic.load.add.f32.p0(ptr %addr, float %val) ret float %ret @@ -27,14 +36,23 @@ declare float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %v define float @atomic_add_f32_addrspace1(ptr addrspace(1) %addr, float %val) { ; CHECK-LABEL: atomic_add_f32_addrspace1( ; CHECK: { -; CHECK-NEXT: .reg .b32 %r<3>; +; CHECK-NEXT: .reg .pred %p<2>; +; CHECK-NEXT: .reg .b32 %r<5>; ; CHECK-NEXT: .reg .b64 %rd<2>; ; CHECK-EMPTY: ; CHECK-NEXT: // %bb.0: +; CHECK-NEXT: ld.param.b32 %r2, [atomic_add_f32_addrspace1_param_1]; ; CHECK-NEXT: ld.param.b64 %rd1, [atomic_add_f32_addrspace1_param_0]; -; CHECK-NEXT: ld.param.b32 %r1, [atomic_add_f32_addrspace1_param_1]; -; CHECK-NEXT: atom.global.add.f32 %r2, [%rd1], %r1; -; CHECK-NEXT: st.param.b32 [func_retval0], %r2; +; CHECK-NEXT: ld.volatile.global.b32 %r4, [%rd1]; +; CHECK-NEXT: $L__BB1_1: // %atomicrmw.start +; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: add.rn.f32 %r3, %r4, %r2; +; CHECK-NEXT: atom.global.cas.b32 %r1, [%rd1], %r4, %r3; +; CHECK-NEXT: setp.ne.b32 %p1, %r1, %r4; +; CHECK-NEXT: mov.b32 %r4, %r1; +; CHECK-NEXT: @%p1 bra $L__BB1_1; +; CHECK-NEXT: // %bb.2: // %atomicrmw.end +; CHECK-NEXT: st.param.b32 [func_retval0], %r1; ; CHECK-NEXT: ret; %ret = call float @llvm.nvvm.atomic.load.add.f32.p1(ptr addrspace(1) %addr, float %val) ret float %ret