squiid/llvm
1
0
Fork 0
Code Issues Pull requests Activity

Recommit "[SCEV] Look through single value PHIs." (take 3)

Browse source 
This reverts commit 1fbdbb5595.

All known issues surfaced by this patch should have been fixed now.

The fixes included fixing issues with SCEV expansion in LV and DA's
reliance on LCSSA phis.
This commit is contained in:
Florian Hahn 2022-06-09 15:20:10 +01:00
parent dd38caf3b5
commit 20d798bd47
No known key found for this signature in database
GPG key ID: EEF712BB5E80EBBA
9 changed files with 55 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
llvm/lib/Analysis/ScalarEvolution.cpp
View file

@ -5927,13 +5927,8 @@ const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
if (const SCEV *S = createNodeFromSelectLikePHI(PN))
return S;
// If the PHI has a single incoming value, follow that value, unless the
// PHI's incoming blocks are in a different loop, in which case doing so
// risks breaking LCSSA form. Instcombine would normally zap these, but
// it doesn't have DominatorTree information, so it may miss cases.
if (Value *V = simplifyInstruction(PN, {getDataLayout(), &TLI, &DT, &AC}))
if (LI.replacementPreservesLCSSAForm(PN, V))
return getSCEV(V);
return getSCEV(V);
// If it's not a loop phi, we can't handle it yet.
return getUnknown(PN);

2
llvm/test/Analysis/DependenceAnalysis/lcssa.ll
View file

@ -2,7 +2,7 @@
; RUN: "-aa-pipeline=basic-aa,tbaa" 2>&1 | FileCheck %s
; CHECK: Src: %v = load i32, i32* %arrayidx1, align 4 --> Dst: store i32 %add, i32* %a.lcssa, align 4
; CHECK-NEXT: da analyze - confused!
; CHECK-NEXT: da analyze - anti [*|<]!
define void @f(i32 *%a, i32 %n, i64 %n2) {
entry:

4
llvm/test/Analysis/ScalarEvolution/cycled_phis.ll
View file

@ -46,7 +46,7 @@ define void @test_02(i32* %p, i32* %q) {
; CHECK-NEXT: %inner_cond = call i1 @cond()
; CHECK-NEXT: --> %inner_cond U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %inner_loop: Variant, %outer_loop: Variant }
; CHECK-NEXT: %inner_lcssa = phi i32 [ %inner_phi, %inner_loop ]
; CHECK-NEXT: --> %inner_lcssa U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Invariant }
; CHECK-NEXT: --> %inner_phi U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Variant }
; CHECK-NEXT: %outer_cond = call i1 @cond()
; CHECK-NEXT: --> %outer_cond U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Invariant }
; CHECK-NEXT: Determining loop execution counts for: @test_02
@ -97,7 +97,7 @@ define void @test_03(i32* %p, i32* %q) {
; CHECK-NEXT: %inner_cond = call i1 @cond()
; CHECK-NEXT: --> %inner_cond U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %inner_loop: Variant, %outer_loop: Variant }
; CHECK-NEXT: %inner_lcssa = phi i32 [ %inner_phi_1, %inner_loop ]
; CHECK-NEXT: --> %inner_lcssa U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Invariant }
; CHECK-NEXT: --> %inner_phi_1 U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Variant }
; CHECK-NEXT: %outer_cond = call i1 @cond()
; CHECK-NEXT: --> %outer_cond U: full-set S: full-set Exits: <<Unknown>> LoopDispositions: { %outer_loop: Variant, %inner_loop: Invariant }
; CHECK-NEXT: Determining loop execution counts for: @test_03

2
llvm/test/Analysis/ScalarEvolution/incorrect-exit-count.ll
View file

@ -53,7 +53,7 @@ define dso_local i32 @f() {
; CHECK-NEXT: %dec.3 = add nsw i32 %storemerge1921.3, -1
; CHECK-NEXT: --> {2,+,-1}<nsw><%inner.loop> U: [2,3) S: [2,3) Exits: <<Unknown>> LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant }
; CHECK-NEXT: %storemerge1921.lcssa25.3 = phi i32 [ %storemerge1921.3, %for.end.3 ]
; CHECK-NEXT: --> %storemerge1921.lcssa25.3 U: [3,4) S: [3,4) Exits: <<Unknown>> LoopDispositions: { %outer.loop: Variant, %for.cond6: Invariant, %inner.loop: Invariant }
; CHECK-NEXT: --> {3,+,-1}<nuw><nsw><%inner.loop> U: [3,4) S: [3,4) Exits: <<Unknown>> LoopDispositions: { %outer.loop: Variant, %for.cond6: Variant, %inner.loop: Computable }
; CHECK-NEXT: %dec16 = add nsw i32 %storemerge23, -1
; CHECK-NEXT: --> {2,+,-1}<nsw><%outer.loop> U: [0,3) S: [0,3) Exits: <<Unknown>> LoopDispositions: { %outer.loop: Computable, %for.cond6: Invariant, %inner.loop: Invariant }
; CHECK-NEXT: Determining loop execution counts for: @f

4
llvm/test/Analysis/ScalarEvolution/solve-quadratic-i1.ll
View file

@ -59,9 +59,9 @@ define void @f1() #0 {
; CHECK-NEXT: %v6 = add nuw nsw i32 %v1, 1
; CHECK-NEXT: --> {4,+,1}<nuw><nsw><%b1> U: [4,7) S: [4,7) Exits: 6 LoopDispositions: { %b1: Computable }
; CHECK-NEXT: %v7 = phi i32 [ %v1, %b1 ]
; CHECK-NEXT: --> %v7 U: [3,6) S: [3,6) --> 5 U: [5,6) S: [5,6)
; CHECK-NEXT: --> {3,+,1}<nuw><nsw><%b1> U: [3,6) S: [3,6) --> 5 U: [5,6) S: [5,6)
; CHECK-NEXT: %v8 = phi i16 [ %v3, %b1 ]
; CHECK-NEXT: --> %v8 U: full-set S: full-set --> 12 U: [12,13) S: [12,13)
; CHECK-NEXT: --> {3,+,4,+,1}<%b1> U: full-set S: full-set --> 12 U: [12,13) S: [12,13)
; CHECK-NEXT: Determining loop execution counts for: @f1
; CHECK-NEXT: Loop %b3: <multiple exits> Unpredictable backedge-taken count.
; CHECK-NEXT: Loop %b3: Unpredictable max backedge-taken count.

6
llvm/test/Analysis/ScalarEvolution/solve-quadratic-overflow.ll
View file

@ -12,11 +12,11 @@
; CHECK-NEXT: %v3 = mul i16 %v2, %v2
; CHECK-NEXT: --> {1,+,3,+,2}<%b1> U: full-set S: full-set Exits: 0 LoopDispositions: { %b1: Computable }
; CHECK-NEXT: %v5 = phi i16 [ %v2, %b1 ]
; CHECK-NEXT: --> %v5 U: [-256,0) S: [-256,0)
; CHECK-NEXT: --> {-1,+,-1}<%b1> U: [-256,0) S: [-256,0) --> -256 U: [-256,-255) S: [-256,-255)
; CHECK-NEXT: %v6 = phi i16 [ %v3, %b1 ]
; CHECK-NEXT: --> %v6 U: full-set S: full-set
; CHECK-NEXT: --> {1,+,3,+,2}<%b1> U: full-set S: full-set --> 0 U: [0,1) S: [0,1)
; CHECK-NEXT: %v7 = sext i16 %v5 to i32
; CHECK-NEXT: --> (sext i16 %v5 to i32) U: [-256,0) S: [-256,0)
; CHECK-NEXT: --> {-1,+,-1}<nsw><%b1> U: [-256,0) S: [-256,0) --> -256 U: [-256,-255) S: [-256,-255)
; CHECK-NEXT: Determining loop execution counts for: @f0
; CHECK-NEXT: Loop %b1: backedge-taken count is 255
; CHECK-NEXT: Loop %b1: max backedge-taken count is 255

2
llvm/test/Analysis/ScalarEvolution/trivial-phis.ll
View file

@ -2,7 +2,7 @@
; CHECK-LABEL: @test1
; CHECK: %add.lcssa.wide = phi i64 [ %indvars.iv.next, %do.body ]
; CHECK-NEXT: --> %add.lcssa.wide U: [1,2147483648) S: [1,2147483648)
; CHECK-NEXT: --> {1,+,1}<nuw><nsw><%do.body> U: [1,2147483648) S: [1,2147483648)
define i64 @test1(i32 signext %n, float* %A) {
entry:

40
llvm/test/Transforms/LoopStrengthReduce/funclet.ll
View file

@ -15,23 +15,21 @@ define void @f() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[THROW:%.*]]
; CHECK: throw:
; CHECK-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
; CHECK-NEXT: invoke void @reserve()
; CHECK-NEXT: to label [[THROW]] unwind label [[PAD:%.*]]
; CHECK: pad:
; CHECK-NEXT: [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
; CHECK-NEXT: [[CS:%.*]] = catchswitch within none [label %unreachable] unwind label [[BLAH2:%.*]]
; CHECK: unreachable:
; CHECK-NEXT: [[TMP0:%.*]] = catchpad within [[CS]] []
; CHECK-NEXT: unreachable
; CHECK: blah2:
; CHECK-NEXT: [[CLEANUPPADI4_I_I_I:%.*]] = cleanuppad within none []
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[PHI2]], i32 -1
; CHECK-NEXT: br label [[LOOP_BODY:%.*]]
; CHECK: loop_body:
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP1:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLAH2]] ]
; CHECK-NEXT: [[SCEVGEP1]] = getelementptr i8, i8* [[LSR_IV]], i32 1
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP1]], undef
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[ITER:%.*]] ], [ 0, [[BLAH2]] ]
; CHECK-NEXT: [[LSR_IV_NEXT]] = add nuw nsw i32 [[LSR_IV]], -1
; CHECK-NEXT: [[LSR_IV_NEXT1:%.*]] = inttoptr i32 [[LSR_IV_NEXT]] to i8*
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[LSR_IV_NEXT1]], null
; CHECK-NEXT: br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
; CHECK: iter:
; CHECK-NEXT: br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
@ -76,27 +74,25 @@ define void @g() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[THROW:%.*]]
; CHECK: throw:
; CHECK-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
; CHECK-NEXT: invoke void @reserve()
; CHECK-NEXT: to label [[THROW]] unwind label [[PAD:%.*]]
; CHECK: pad:
; CHECK-NEXT: [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
; CHECK-NEXT: [[CS:%.*]] = catchswitch within none [label [[UNREACHABLE:%.*]], label %blah] unwind to caller
; CHECK: unreachable:
; CHECK-NEXT: [[TMP0:%.*]] = catchpad within [[CS]] []
; CHECK-NEXT: unreachable
; CHECK: blah:
; CHECK-NEXT: [[CATCHPAD:%.*]] = catchpad within [[CS]] []
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[PHI2]], i32 -1
; CHECK-NEXT: br label [[LOOP_BODY:%.*]]
; CHECK: unwind_out:
; CHECK-NEXT: catchret from [[CATCHPAD]] to label [[LEAVE:%.*]]
; CHECK: leave:
; CHECK-NEXT: ret void
; CHECK: loop_body:
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP1:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLAH:%.*]] ]
; CHECK-NEXT: [[SCEVGEP1]] = getelementptr i8, i8* [[LSR_IV]], i32 1
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP1]], undef
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[ITER:%.*]] ], [ 0, [[BLAH:%.*]] ]
; CHECK-NEXT: [[LSR_IV_NEXT]] = add nuw nsw i32 [[LSR_IV]], -1
; CHECK-NEXT: [[LSR_IV_NEXT1:%.*]] = inttoptr i32 [[LSR_IV_NEXT]] to i8*
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[LSR_IV_NEXT1]], null
; CHECK-NEXT: br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
; CHECK: iter:
; CHECK-NEXT: br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
@ -142,7 +138,6 @@ define void @h() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[THROW:%.*]]
; CHECK: throw:
; CHECK-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
; CHECK-NEXT: invoke void @reserve()
; CHECK-NEXT: to label [[THROW]] unwind label [[PAD:%.*]]
; CHECK: pad:
@ -151,18 +146,17 @@ define void @h() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: [[TMP0:%.*]] = catchpad within [[CS]] []
; CHECK-NEXT: unreachable
; CHECK: blug:
; CHECK-NEXT: [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[PAD]] ]
; CHECK-NEXT: [[CATCHPAD:%.*]] = catchpad within [[CS]] []
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[PHI2]], i32 -1
; CHECK-NEXT: br label [[LOOP_BODY:%.*]]
; CHECK: unwind_out:
; CHECK-NEXT: catchret from [[CATCHPAD]] to label [[LEAVE:%.*]]
; CHECK: leave:
; CHECK-NEXT: ret void
; CHECK: loop_body:
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP1:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[BLUG:%.*]] ]
; CHECK-NEXT: [[SCEVGEP1]] = getelementptr i8, i8* [[LSR_IV]], i32 1
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP1]], undef
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[ITER:%.*]] ], [ 0, [[BLUG:%.*]] ]
; CHECK-NEXT: [[LSR_IV_NEXT]] = add nuw nsw i32 [[LSR_IV]], -1
; CHECK-NEXT: [[LSR_IV_NEXT1:%.*]] = inttoptr i32 [[LSR_IV_NEXT]] to i8*
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[LSR_IV_NEXT1]], null
; CHECK-NEXT: br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
; CHECK: iter:
; CHECK-NEXT: br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]
@ -208,11 +202,9 @@ define void @i() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[THROW:%.*]]
; CHECK: throw:
; CHECK-NEXT: [[TMP96:%.*]] = getelementptr inbounds i8, i8* undef, i32 1
; CHECK-NEXT: invoke void @reserve()
; CHECK-NEXT: to label [[THROW]] unwind label [[CATCHPAD:%.*]]
; CHECK: catchpad:
; CHECK-NEXT: [[PHI2:%.*]] = phi i8* [ [[TMP96]], [[THROW]] ]
; CHECK-NEXT: [[CS:%.*]] = catchswitch within none [label %cp_body] unwind label [[CLEANUPPAD:%.*]]
; CHECK: cp_body:
; CHECK-NEXT: [[TMP0:%.*]] = catchpad within [[CS]] []
@ -221,12 +213,12 @@ define void @i() personality i32 (...)* @_except_handler3 {
; CHECK-NEXT: [[TMP1:%.*]] = cleanuppad within none []
; CHECK-NEXT: br label [[LOOP_HEAD]]
; CHECK: loop_head:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[PHI2]], i32 -1
; CHECK-NEXT: br label [[LOOP_BODY:%.*]]
; CHECK: loop_body:
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i8* [ [[SCEVGEP1:%.*]], [[ITER:%.*]] ], [ [[SCEVGEP]], [[LOOP_HEAD]] ]
; CHECK-NEXT: [[SCEVGEP1]] = getelementptr i8, i8* [[LSR_IV]], i32 1
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[SCEVGEP1]], undef
; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[ITER:%.*]] ], [ 0, [[LOOP_HEAD]] ]
; CHECK-NEXT: [[LSR_IV_NEXT]] = add nuw nsw i32 [[LSR_IV]], -1
; CHECK-NEXT: [[LSR_IV_NEXT1:%.*]] = inttoptr i32 [[LSR_IV_NEXT]] to i8*
; CHECK-NEXT: [[TMP100:%.*]] = icmp eq i8* [[LSR_IV_NEXT1]], null
; CHECK-NEXT: br i1 [[TMP100]], label [[UNWIND_OUT:%.*]], label [[ITER]]
; CHECK: iter:
; CHECK-NEXT: br i1 true, label [[UNWIND_OUT]], label [[LOOP_BODY]]

55
llvm/test/Transforms/LoopVectorize/pr45259.ll
View file

@ -6,7 +6,7 @@
define i8 @widget(i8* %arr, i8 %t9) {
; CHECK-LABEL: @widget(
; CHECK-NEXT: bb:
; CHECK-NEXT: [[ARR2:%.*]] = ptrtoint i8* [[ARR:%.*]] to i64
; CHECK-NEXT: [[ARR1:%.*]] = ptrtoint i8* [[ARR:%.*]] to i64
; CHECK-NEXT: br label [[BB6:%.*]]
; CHECK: bb6:
; CHECK-NEXT: [[T1_0:%.*]] = phi i8* [ [[ARR]], [[BB:%.*]] ], [ null, [[BB6]] ]
@ -14,24 +14,25 @@ define i8 @widget(i8* %arr, i8 %t9) {
; CHECK-NEXT: br i1 [[C]], label [[FOR_PREHEADER:%.*]], label [[BB6]]
; CHECK: for.preheader:
; CHECK-NEXT: [[T1_0_LCSSA:%.*]] = phi i8* [ [[T1_0]], [[BB6]] ]
; CHECK-NEXT: [[T1_0_LCSSA1:%.*]] = ptrtoint i8* [[T1_0_LCSSA]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[T1_0_LCSSA1]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[ARR2]] to i32
; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP2]], 4
; CHECK-NEXT: [[T1_0_LCSSA2:%.*]] = ptrtoint i8* [[T1_0_LCSSA]] to i64
; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[ARR1]] to i32
; CHECK-NEXT: [[TMP1:%.*]] = sub i32 0, [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[T1_0_LCSSA2]] to i32
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[TMP1]], [[TMP2]]
; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 4
; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
; CHECK: vector.scevcheck:
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[T1_0_LCSSA1]], -1
; CHECK-NEXT: [[TMP4:%.*]] = sub i64 [[TMP3]], [[ARR2]]
; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[TMP4]] to i8
; CHECK-NEXT: [[TMP6:%.*]] = add i8 1, [[TMP5]]
; CHECK-NEXT: [[TMP9:%.*]] = icmp slt i8 [[TMP6]], 1
; CHECK-NEXT: [[TMP11:%.*]] = icmp ugt i64 [[TMP4]], 255
; CHECK-NEXT: [[TMP12:%.*]] = or i1 [[TMP9]], [[TMP11]]
; CHECK-NEXT: br i1 [[TMP12]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK-NEXT: [[TMP4:%.*]] = sub i64 -1, [[ARR1]]
; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], [[T1_0_LCSSA2]]
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i8
; CHECK-NEXT: [[TMP7:%.*]] = add i8 1, [[TMP6]]
; CHECK-NEXT: [[TMP8:%.*]] = icmp slt i8 [[TMP7]], 1
; CHECK-NEXT: [[TMP9:%.*]] = icmp ugt i64 [[TMP5]], 255
; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]]
; CHECK-NEXT: br i1 [[TMP10]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP2]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP3]], 4
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP3]], [[N_MOD_VF]]
; CHECK-NEXT: [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[T9:%.*]], i32 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
@ -39,20 +40,20 @@ define i8 @widget(i8* %arr, i8 %t9) {
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP14:%.*]] = add <4 x i8> [[VEC_IND]], <i8 1, i8 1, i8 1, i8 1>
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i8> [[TMP14]], i32 0
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, i8* [[ARR]], i8 [[TMP15]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp slt <4 x i8> [[TMP14]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP18:%.*]] = zext <4 x i1> [[TMP17]] to <4 x i8>
; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, i8* [[TMP16]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = bitcast i8* [[TMP19]] to <4 x i8>*
; CHECK-NEXT: store <4 x i8> [[TMP18]], <4 x i8>* [[TMP20]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = add <4 x i8> [[VEC_IND]], <i8 1, i8 1, i8 1, i8 1>
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i8> [[TMP11]], i32 0
; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, i8* [[ARR]], i8 [[TMP12]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp slt <4 x i8> [[TMP11]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP15:%.*]] = zext <4 x i1> [[TMP14]] to <4 x i8>
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, i8* [[TMP13]], i32 0
; CHECK-NEXT: [[TMP17:%.*]] = bitcast i8* [[TMP16]] to <4 x i8>*
; CHECK-NEXT: store <4 x i8> [[TMP15]], <4 x i8>* [[TMP17]], align 1
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND]], <i8 4, i8 4, i8 4, i8 4>
; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i8 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_PREHEADER]] ], [ 0, [[VECTOR_SCEVCHECK]] ]