diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index cab8e8f7987ec..3fb8a160625eb 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -18982,11 +18982,21 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable( IsEqualCostAltShuffleToTrim()) { PreferTrimmedTree |= TotalSubtreeCost == GatherCost; // If the remaining tree is just a buildvector - exit, it will cause - // endless attempts to vectorize. + // endless attempts to vectorize. When the tree is already profitable, + // skip trimming this node and let the post-loop logic (including + // gathered loads processing) decide. if (VectorizableTree.front()->hasState() && VectorizableTree.front()->getOpcode() == Instruction::InsertElement && - TE->Idx == 1) + TE->Idx == 1) { + if (Cost < -SLPCostThreshold) { + LLVM_DEBUG(dbgs() << "SLP: Skipping trim of node " << TE->Idx + << " - tree already profitable with cost " << Cost + << ".\n"); + Worklist.pop(); + continue; + } return InstructionCost::getInvalid(); + } LLVM_DEBUG(dbgs() << "SLP: Trimming unprofitable subtree at node " << TE->Idx << " with cost " diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll index 9b45fe6a2804b..174be55bd7b0a 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll @@ -282,26 +282,23 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) { define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) { ; SSE-LABEL: @ashr_lshr_shl_v8i32( -; SSE-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 4 -; SSE-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5 -; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6 +; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6 ; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7 -; SSE-NEXT: [[B4:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 4 -; SSE-NEXT: [[B5:%.*]] = extractelement <8 x i32> [[B]], i32 5 -; SSE-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6 +; SSE-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6 ; SSE-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7 ; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> ; SSE-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> ; SSE-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]] ; SSE-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]] ; SSE-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> -; SSE-NEXT: [[AB4:%.*]] = lshr i32 [[A4]], [[B4]] -; SSE-NEXT: [[AB5:%.*]] = lshr i32 [[A5]], [[B5]] +; SSE-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> +; SSE-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> +; SSE-NEXT: [[TMP9:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP7]] ; SSE-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]] ; SSE-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]] ; SSE-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> -; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[AB4]], i32 4 -; SSE-NEXT: [[R51:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5 +; SSE-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <8 x i32> +; SSE-NEXT: [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP10]], <8 x i32> ; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6 ; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7 ; SSE-NEXT: ret <8 x i32> [[R7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll index d812cc813c20f..8a5e278e2b03a 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll @@ -282,26 +282,23 @@ define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) { define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) { ; SSE-LABEL: @ashr_lshr_shl_v8i32( -; SSE-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 4 -; SSE-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5 -; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6 +; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6 ; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7 -; SSE-NEXT: [[B4:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 4 -; SSE-NEXT: [[B5:%.*]] = extractelement <8 x i32> [[B]], i32 5 -; SSE-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6 +; SSE-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6 ; SSE-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7 ; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> ; SSE-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> ; SSE-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]] ; SSE-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]] ; SSE-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> -; SSE-NEXT: [[AB4:%.*]] = lshr i32 [[A4]], [[B4]] -; SSE-NEXT: [[AB5:%.*]] = lshr i32 [[A5]], [[B5]] +; SSE-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> +; SSE-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <2 x i32> +; SSE-NEXT: [[TMP9:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP7]] ; SSE-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]] ; SSE-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]] ; SSE-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> poison, <8 x i32> -; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[AB4]], i32 4 -; SSE-NEXT: [[R51:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5 +; SSE-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <8 x i32> +; SSE-NEXT: [[R51:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> [[TMP10]], <8 x i32> ; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R51]], i32 [[AB6]], i32 6 ; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7 ; SSE-NEXT: ret <8 x i32> [[R7]] diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll index 57deca1d62516..7f7e77eadc987 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll @@ -607,38 +607,25 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) { ; SSE-NEXT: ret <8 x double> [[TMP1]] ; ; SLM-LABEL: @buildvector_div_8f64( -; SLM-NEXT: [[A0:%.*]] = extractelement <8 x double> [[A:%.*]], i32 0 -; SLM-NEXT: [[A1:%.*]] = extractelement <8 x double> [[A]], i32 1 -; SLM-NEXT: [[A2:%.*]] = extractelement <8 x double> [[A]], i32 2 -; SLM-NEXT: [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3 -; SLM-NEXT: [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4 -; SLM-NEXT: [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5 -; SLM-NEXT: [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6 -; SLM-NEXT: [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7 -; SLM-NEXT: [[B0:%.*]] = extractelement <8 x double> [[B:%.*]], i32 0 -; SLM-NEXT: [[B1:%.*]] = extractelement <8 x double> [[B]], i32 1 -; SLM-NEXT: [[B2:%.*]] = extractelement <8 x double> [[B]], i32 2 -; SLM-NEXT: [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3 -; SLM-NEXT: [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4 -; SLM-NEXT: [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5 -; SLM-NEXT: [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6 -; SLM-NEXT: [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7 -; SLM-NEXT: [[C0:%.*]] = fdiv double [[A0]], [[B0]] -; SLM-NEXT: [[C1:%.*]] = fdiv double [[A1]], [[B1]] -; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]] -; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]] -; SLM-NEXT: [[C4:%.*]] = fdiv double [[A4]], [[B4]] -; SLM-NEXT: [[C5:%.*]] = fdiv double [[A5]], [[B5]] -; SLM-NEXT: [[C6:%.*]] = fdiv double [[A6]], [[B6]] -; SLM-NEXT: [[C7:%.*]] = fdiv double [[A7]], [[B7]] -; SLM-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[C0]], i32 0 -; SLM-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[C1]], i32 1 -; SLM-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[C2]], i32 2 -; SLM-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3 -; SLM-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4 -; SLM-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5 -; SLM-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6 -; SLM-NEXT: [[R73:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7 +; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP3:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP2]] +; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP6:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP5]] +; SLM-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP9:%.*]] = fdiv <2 x double> [[TMP7]], [[TMP8]] +; SLM-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP12:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]] +; SLM-NEXT: [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R31:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <8 x i32> +; SLM-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R52:%.*]] = shufflevector <8 x double> [[R31]], <8 x double> [[TMP15]], <8 x i32> +; SLM-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R73:%.*]] = shufflevector <8 x double> [[R52]], <8 x double> [[TMP16]], <8 x i32> ; SLM-NEXT: ret <8 x double> [[R73]] ; ; AVX-LABEL: @buildvector_div_8f64( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll index d1a5c3bb032e0..8b8bc71c2ceda 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll @@ -607,38 +607,25 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) { ; SSE-NEXT: ret <8 x double> [[TMP1]] ; ; SLM-LABEL: @buildvector_div_8f64( -; SLM-NEXT: [[A0:%.*]] = extractelement <8 x double> [[A:%.*]], i32 0 -; SLM-NEXT: [[A1:%.*]] = extractelement <8 x double> [[A]], i32 1 -; SLM-NEXT: [[A2:%.*]] = extractelement <8 x double> [[A]], i32 2 -; SLM-NEXT: [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3 -; SLM-NEXT: [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4 -; SLM-NEXT: [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5 -; SLM-NEXT: [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6 -; SLM-NEXT: [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7 -; SLM-NEXT: [[B0:%.*]] = extractelement <8 x double> [[B:%.*]], i32 0 -; SLM-NEXT: [[B1:%.*]] = extractelement <8 x double> [[B]], i32 1 -; SLM-NEXT: [[B2:%.*]] = extractelement <8 x double> [[B]], i32 2 -; SLM-NEXT: [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3 -; SLM-NEXT: [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4 -; SLM-NEXT: [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5 -; SLM-NEXT: [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6 -; SLM-NEXT: [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7 -; SLM-NEXT: [[C0:%.*]] = fdiv double [[A0]], [[B0]] -; SLM-NEXT: [[C1:%.*]] = fdiv double [[A1]], [[B1]] -; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]] -; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]] -; SLM-NEXT: [[C4:%.*]] = fdiv double [[A4]], [[B4]] -; SLM-NEXT: [[C5:%.*]] = fdiv double [[A5]], [[B5]] -; SLM-NEXT: [[C6:%.*]] = fdiv double [[A6]], [[B6]] -; SLM-NEXT: [[C7:%.*]] = fdiv double [[A7]], [[B7]] -; SLM-NEXT: [[R0:%.*]] = insertelement <8 x double> undef, double [[C0]], i32 0 -; SLM-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[C1]], i32 1 -; SLM-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[C2]], i32 2 -; SLM-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3 -; SLM-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4 -; SLM-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5 -; SLM-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6 -; SLM-NEXT: [[R73:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7 +; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A:%.*]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP3:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP2]] +; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP6:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP5]] +; SLM-NEXT: [[TMP7:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP8:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP9:%.*]] = fdiv <2 x double> [[TMP7]], [[TMP8]] +; SLM-NEXT: [[TMP10:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP11:%.*]] = shufflevector <8 x double> [[B]], <8 x double> poison, <2 x i32> +; SLM-NEXT: [[TMP12:%.*]] = fdiv <2 x double> [[TMP10]], [[TMP11]] +; SLM-NEXT: [[TMP13:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[TMP14:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R31:%.*]] = shufflevector <8 x double> [[TMP13]], <8 x double> [[TMP14]], <8 x i32> +; SLM-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R52:%.*]] = shufflevector <8 x double> [[R31]], <8 x double> [[TMP15]], <8 x i32> +; SLM-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> poison, <8 x i32> +; SLM-NEXT: [[R73:%.*]] = shufflevector <8 x double> [[R52]], <8 x double> [[TMP16]], <8 x i32> ; SLM-NEXT: ret <8 x double> [[R73]] ; ; AVX-LABEL: @buildvector_div_8f64( diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll index 1bb24c524bb3e..042f5cb3f512b 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll @@ -5,8 +5,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-LABEL: define void @test( ; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]], i64 [[ARG2:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[BB:.*]]: -; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> , i32 [[ARG]], i32 2 -; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ARG1]], i32 3 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> , i32 [[ARG]], i32 3 ; CHECK-NEXT: br label %[[BB3:.*]] ; CHECK: [[BB3]]: ; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ] @@ -21,7 +20,6 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP5]], i32 0 ; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP4]] to i64 ; CHECK-NEXT: [[TRUNC10:%.*]] = trunc i64 [[TMP10]] to i32 -; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1 ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP5]], i32 2 ; CHECK-NEXT: [[TMP7:%.*]] = zext i32 [[TMP6]] to i64 ; CHECK-NEXT: [[TRUNC27:%.*]] = trunc i64 [[TMP7]] to i32 @@ -30,14 +28,15 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) { ; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]] ; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]] ; CHECK-NEXT: [[SHL35:%.*]] = shl i32 [[ARG1]], 0 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> poison, i32 [[TRUNC10]], i32 0 -; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[SHL]], i32 1 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TRUNC27]], i32 2 -; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TRUNC19]], i32 3 +; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]] +; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i32 0 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i32 1 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TRUNC27]], i32 3 +; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> ; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]] -; CHECK-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> poison, i32 [[SHL]], i32 0 -; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <8 x i32> [[TMP23]], <8 x i32> [[TMP27]], <8 x i32> +; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i32 5 ; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[SHL35]], i32 6 ; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i32 7 ; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll index e67589426bcc6..e2977935c167c 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/scalarize-ctlz.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64 %s | FileCheck %s --check-prefixes=SSE -; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v2 %s | FileCheck %s --check-prefixes=SSE +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64 %s | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v2 %s | FileCheck %s --check-prefixes=SSE,SSE4_2 ; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v3 %s | FileCheck %s --check-prefixes=AVX2 ; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-- -mcpu=x86-64-v4 %s | FileCheck %s --check-prefixes=AVX512 @@ -95,33 +95,52 @@ define <4 x i64> @scalarize_ctlz_v4i64(<4 x i64> %v) { } define <8 x i64> @scalarize_ctlz_v8i64(<8 x i64> %v) { -; SSE-LABEL: define <8 x i64> @scalarize_ctlz_v8i64( -; SSE-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] { -; SSE-NEXT: [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0 -; SSE-NEXT: [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1 -; SSE-NEXT: [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2 -; SSE-NEXT: [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3 -; SSE-NEXT: [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4 -; SSE-NEXT: [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5 -; SSE-NEXT: [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6 -; SSE-NEXT: [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7 -; SSE-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false) -; SSE-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false) -; SSE-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false) -; SSE-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false) -; SSE-NEXT: [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false) -; SSE-NEXT: [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false) -; SSE-NEXT: [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false) -; SSE-NEXT: [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false) -; SSE-NEXT: [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0 -; SSE-NEXT: [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1 -; SSE-NEXT: [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2 -; SSE-NEXT: [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3 -; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4 -; SSE-NEXT: [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5 -; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6 -; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7 -; SSE-NEXT: ret <8 x i64> [[R7]] +; SSE2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64( +; SSE2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] { +; SSE2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> +; SSE2-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP1]], i1 false) +; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> +; SSE2-NEXT: [[TMP4:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP3]], i1 false) +; SSE2-NEXT: [[TMP5:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> +; SSE2-NEXT: [[TMP6:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP5]], i1 false) +; SSE2-NEXT: [[TMP7:%.*]] = shufflevector <8 x i64> [[V]], <8 x i64> poison, <2 x i32> +; SSE2-NEXT: [[TMP8:%.*]] = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> [[TMP7]], i1 false) +; SSE2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <8 x i32> +; SSE2-NEXT: [[TMP10:%.*]] = shufflevector <2 x i64> [[TMP4]], <2 x i64> poison, <8 x i32> +; SSE2-NEXT: [[R31:%.*]] = shufflevector <8 x i64> [[TMP9]], <8 x i64> [[TMP10]], <8 x i32> +; SSE2-NEXT: [[TMP11:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> poison, <8 x i32> +; SSE2-NEXT: [[R52:%.*]] = shufflevector <8 x i64> [[R31]], <8 x i64> [[TMP11]], <8 x i32> +; SSE2-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP8]], <2 x i64> poison, <8 x i32> +; SSE2-NEXT: [[R73:%.*]] = shufflevector <8 x i64> [[R52]], <8 x i64> [[TMP12]], <8 x i32> +; SSE2-NEXT: ret <8 x i64> [[R73]] +; +; SSE4_2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64( +; SSE4_2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] { +; SSE4_2-NEXT: [[V0:%.*]] = extractelement <8 x i64> [[V]], i64 0 +; SSE4_2-NEXT: [[V1:%.*]] = extractelement <8 x i64> [[V]], i64 1 +; SSE4_2-NEXT: [[V2:%.*]] = extractelement <8 x i64> [[V]], i64 2 +; SSE4_2-NEXT: [[V3:%.*]] = extractelement <8 x i64> [[V]], i64 3 +; SSE4_2-NEXT: [[V4:%.*]] = extractelement <8 x i64> [[V]], i64 4 +; SSE4_2-NEXT: [[V5:%.*]] = extractelement <8 x i64> [[V]], i64 5 +; SSE4_2-NEXT: [[V6:%.*]] = extractelement <8 x i64> [[V]], i64 6 +; SSE4_2-NEXT: [[V7:%.*]] = extractelement <8 x i64> [[V]], i64 7 +; SSE4_2-NEXT: [[C0:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V0]], i1 false) +; SSE4_2-NEXT: [[C1:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V1]], i1 false) +; SSE4_2-NEXT: [[C2:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V2]], i1 false) +; SSE4_2-NEXT: [[C3:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V3]], i1 false) +; SSE4_2-NEXT: [[C4:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V4]], i1 false) +; SSE4_2-NEXT: [[C5:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V5]], i1 false) +; SSE4_2-NEXT: [[C6:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V6]], i1 false) +; SSE4_2-NEXT: [[C7:%.*]] = tail call range(i64 0, 65) i64 @llvm.ctlz.i64(i64 [[V7]], i1 false) +; SSE4_2-NEXT: [[R0:%.*]] = insertelement <8 x i64> poison, i64 [[C0]], i64 0 +; SSE4_2-NEXT: [[R1:%.*]] = insertelement <8 x i64> [[R0]], i64 [[C1]], i64 1 +; SSE4_2-NEXT: [[R2:%.*]] = insertelement <8 x i64> [[R1]], i64 [[C2]], i64 2 +; SSE4_2-NEXT: [[R3:%.*]] = insertelement <8 x i64> [[R2]], i64 [[C3]], i64 3 +; SSE4_2-NEXT: [[R4:%.*]] = insertelement <8 x i64> [[R3]], i64 [[C4]], i64 4 +; SSE4_2-NEXT: [[R5:%.*]] = insertelement <8 x i64> [[R4]], i64 [[C5]], i64 5 +; SSE4_2-NEXT: [[R6:%.*]] = insertelement <8 x i64> [[R5]], i64 [[C6]], i64 6 +; SSE4_2-NEXT: [[R7:%.*]] = insertelement <8 x i64> [[R6]], i64 [[C7]], i64 7 +; SSE4_2-NEXT: ret <8 x i64> [[R7]] ; ; AVX2-LABEL: define <8 x i64> @scalarize_ctlz_v8i64( ; AVX2-SAME: <8 x i64> [[V:%.*]]) #[[ATTR0]] { diff --git a/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll b/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll index e0f70ea861cb4..4b2b2b8adefa9 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/select-copyable-cmp-poison.ll @@ -13,20 +13,23 @@ define i32 @foo(i32 %v2, ptr byval(%struct.S2) align 8 %v5, i1 %cmp3644) { ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i32> poison, i32 [[V2]], i32 1 ; CHECK-NEXT: br label %[[BB5:.*]] ; CHECK: [[BB5]]: +; CHECK-NEXT: [[F2297:%.*]] = getelementptr i8, ptr [[V5]], i64 112 ; CHECK-NEXT: [[F1783:%.*]] = getelementptr i8, ptr [[V5]], i64 108 ; CHECK-NEXT: [[F2354:%.*]] = getelementptr i8, ptr [[V5]], i64 88 ; CHECK-NEXT: [[F2882:%.*]] = getelementptr i8, ptr [[V5]], i64 100 +; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[F2297]], align 8 ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[F1783]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[F2882]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[F2354]], align 8 +; CHECK-NEXT: [[DOTFR5712:%.*]] = freeze i32 [[TMP7]] ; CHECK-NEXT: [[DOTFR5727:%.*]] = freeze i32 [[TMP2]] -; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr [[F2882]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <2 x i32> +; CHECK-NEXT: [[TMP31:%.*]] = insertelement <2 x i32> poison, i32 [[TMP3]], i32 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP31]], i32 [[TMP1]], i32 1 ; CHECK-NEXT: [[TMP5:%.*]] = freeze <2 x i32> [[TMP4]] -; CHECK-NEXT: [[DOTFR5726:%.*]] = freeze i32 [[TMP1]] ; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP0]], i32 [[DOTFR5727]], i32 0 -; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> -; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <8 x i32> [[TMP6]], <8 x i32> [[TMP7]], <8 x i32> -; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP31]], i32 [[DOTFR5726]], i32 6 +; CHECK-NEXT: [[TMP36:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[DOTFR5712]], i32 3 +; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <8 x i32> +; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <8 x i32> [[TMP36]], <8 x i32> [[TMP37]], <8 x i32> ; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> poison, <8 x i32> ; CHECK-NEXT: [[TMP34:%.*]] = srem <8 x i32> [[TMP33]], splat (i32 46337) ; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <8 x i32> [[TMP34]], <8 x i32> poison, <8 x i32> diff --git a/llvm/test/Transforms/SLPVectorizer/X86/trunced-buildvector-scalar-extended.ll b/llvm/test/Transforms/SLPVectorizer/X86/trunced-buildvector-scalar-extended.ll index 6623e9b8ecc84..11b76256c42a4 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/trunced-buildvector-scalar-extended.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/trunced-buildvector-scalar-extended.ll @@ -7,14 +7,10 @@ define <4 x float> @test(i64 %0) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> , i64 [[TMP0]], i32 2 ; CHECK-NEXT: [[TMP2:%.*]] = trunc <4 x i64> [[TMP3]] to <4 x i32> -; CHECK-NEXT: [[TMP11:%.*]] = sitofp i64 0 to float -; CHECK-NEXT: [[TMP12:%.*]] = sitofp i64 0 to float ; CHECK-NEXT: [[TMP13:%.*]] = sitofp i64 [[TMP0]] to float ; CHECK-NEXT: [[TMP14:%.*]] = sitofp i64 0 to float ; CHECK-NEXT: [[TMP5:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float> -; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> poison, float [[TMP11]], i32 0 -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[TMP12]], i32 1 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP13]], i32 2 +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> , float [[TMP13]], i32 2 ; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP10]], float [[TMP14]], i32 3 ; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x float> [[TMP4]], [[TMP5]] ; CHECK-NEXT: [[TMP7:%.*]] = fcmp ogt <4 x float> [[TMP6]], zeroinitializer diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll index 4ac93c787c4a9..6a479174777b0 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-widest-phis.ll @@ -18,10 +18,13 @@ define void @foo(i1 %arg) { ; CHECK: bb4: ; CHECK-NEXT: [[TMP4:%.*]] = fpext <4 x float> [[TMP2]] to <4 x double> ; CHECK-NEXT: [[CONV2:%.*]] = uitofp i16 0 to double -; CHECK-NEXT: [[ADD1:%.*]] = fadd double [[TMP3]], [[CONV2]] -; CHECK-NEXT: [[SUB1:%.*]] = fsub double 0.000000e+00, 0.000000e+00 -; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> , double [[SUB1]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP5]], double [[ADD1]], i32 1 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> , double [[TMP3]], i32 1 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> , double [[CONV2]], i32 1 +; CHECK-NEXT: [[TMP7:%.*]] = fsub <2 x double> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x double> [[TMP5]], [[TMP6]] +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> [[TMP8]], <2 x i32> +; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <4 x i32> +; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x double> , <4 x double> [[TMP15]], <4 x i32> ; CHECK-NEXT: [[TMP11:%.*]] = fcmp ogt <4 x double> [[TMP10]], [[TMP4]] ; CHECK-NEXT: [[TMP12:%.*]] = fptrunc <4 x double> [[TMP10]] to <4 x float> ; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x float> [[TMP2]], <4 x float> [[TMP12]]