; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fma | FileCheck %s

define <2 x double> @fmaddsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fmaddsubpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB0_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB0_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmaddsub231pd {{.*#+}} xmm2 = (xmm0 * xmm1) +/- xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB0_2
; CHECK-NEXT:  .LBB0_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

define <2 x double> @fmsubaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fmsubaddpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB1_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB1_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsubadd231pd {{.*#+}} xmm2 = (xmm0 * xmm1) -/+ xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB1_2
; CHECK-NEXT:  .LBB1_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

define <2 x double> @fmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fmaddpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB2_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB2_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmadd231pd {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB2_2
; CHECK-NEXT:  .LBB2_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

define <2 x double> @fmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fmsubpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB3_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB3_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsub231pd {{.*#+}} xmm2 = (xmm0 * xmm1) - xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB3_2
; CHECK-NEXT:  .LBB3_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

define <2 x double> @fnmaddpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fnmaddpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB4_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB4_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmadd231pd {{.*#+}} xmm2 = -(xmm0 * xmm1) + xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB4_2
; CHECK-NEXT:  .LBB4_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

define <2 x double> @fnmsubpd_loop_128(i32 %iter, <2 x double> %a, <2 x double> %b, <2 x double> %c) {
; CHECK-LABEL: fnmsubpd_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB5_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB5_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmsub231pd {{.*#+}} xmm2 = -(xmm0 * xmm1) - xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB5_2
; CHECK-NEXT:  .LBB5_3: # %for.end
; CHECK-NEXT:    vmovapd %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <2 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double> %a, <2 x double> %b, <2 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <2 x double> %c.addr.0
}

declare <2 x double> @llvm.x86.fma.vfmaddsub.pd(<2 x double>, <2 x double>, <2 x double>)
declare <2 x double> @llvm.x86.fma.vfmsubadd.pd(<2 x double>, <2 x double>, <2 x double>)
declare <2 x double> @llvm.x86.fma.vfmadd.pd(<2 x double>, <2 x double>, <2 x double>)
declare <2 x double> @llvm.x86.fma.vfmsub.pd(<2 x double>, <2 x double>, <2 x double>)
declare <2 x double> @llvm.x86.fma.vfnmadd.pd(<2 x double>, <2 x double>, <2 x double>)
declare <2 x double> @llvm.x86.fma.vfnmsub.pd(<2 x double>, <2 x double>, <2 x double>)

define <4 x float> @fmaddsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fmaddsubps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB6_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB6_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmaddsub231ps {{.*#+}} xmm2 = (xmm0 * xmm1) +/- xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB6_2
; CHECK-NEXT:  .LBB6_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

define <4 x float> @fmsubaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fmsubaddps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB7_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB7_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsubadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) -/+ xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB7_2
; CHECK-NEXT:  .LBB7_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

define <4 x float> @fmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fmaddps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB8_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB8_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmadd231ps {{.*#+}} xmm2 = (xmm0 * xmm1) + xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB8_2
; CHECK-NEXT:  .LBB8_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

define <4 x float> @fmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fmsubps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB9_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB9_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsub231ps {{.*#+}} xmm2 = (xmm0 * xmm1) - xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB9_2
; CHECK-NEXT:  .LBB9_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

define <4 x float> @fnmaddps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fnmaddps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB10_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB10_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmadd231ps {{.*#+}} xmm2 = -(xmm0 * xmm1) + xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB10_2
; CHECK-NEXT:  .LBB10_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

define <4 x float> @fnmsubps_loop_128(i32 %iter, <4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: fnmsubps_loop_128:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB11_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB11_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmsub231ps {{.*#+}} xmm2 = -(xmm0 * xmm1) - xmm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB11_2
; CHECK-NEXT:  .LBB11_3: # %for.end
; CHECK-NEXT:    vmovaps %xmm2, %xmm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float> %a, <4 x float> %b, <4 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x float> %c.addr.0
}

declare <4 x float> @llvm.x86.fma.vfmaddsub.ps(<4 x float>, <4 x float>, <4 x float>)
declare <4 x float> @llvm.x86.fma.vfmsubadd.ps(<4 x float>, <4 x float>, <4 x float>)
declare <4 x float> @llvm.x86.fma.vfmadd.ps(<4 x float>, <4 x float>, <4 x float>)
declare <4 x float> @llvm.x86.fma.vfmsub.ps(<4 x float>, <4 x float>, <4 x float>)
declare <4 x float> @llvm.x86.fma.vfnmadd.ps(<4 x float>, <4 x float>, <4 x float>)
declare <4 x float> @llvm.x86.fma.vfnmsub.ps(<4 x float>, <4 x float>, <4 x float>)

define <4 x double> @fmaddsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fmaddsubpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB12_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB12_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmaddsub231pd {{.*#+}} ymm2 = (ymm0 * ymm1) +/- ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB12_2
; CHECK-NEXT:  .LBB12_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

define <4 x double> @fmsubaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fmsubaddpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB13_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB13_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsubadd231pd {{.*#+}} ymm2 = (ymm0 * ymm1) -/+ ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB13_2
; CHECK-NEXT:  .LBB13_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

define <4 x double> @fmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fmaddpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB14_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB14_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmadd231pd {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB14_2
; CHECK-NEXT:  .LBB14_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

define <4 x double> @fmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fmsubpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB15_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB15_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsub231pd {{.*#+}} ymm2 = (ymm0 * ymm1) - ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB15_2
; CHECK-NEXT:  .LBB15_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

define <4 x double> @fnmaddpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fnmaddpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB16_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB16_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmadd231pd {{.*#+}} ymm2 = -(ymm0 * ymm1) + ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB16_2
; CHECK-NEXT:  .LBB16_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

define <4 x double> @fnmsubpd_loop_256(i32 %iter, <4 x double> %a, <4 x double> %b, <4 x double> %c) {
; CHECK-LABEL: fnmsubpd_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB17_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB17_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmsub231pd {{.*#+}} ymm2 = -(ymm0 * ymm1) - ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB17_2
; CHECK-NEXT:  .LBB17_3: # %for.end
; CHECK-NEXT:    vmovapd %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <4 x double> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <4 x double> %c.addr.0
}

declare <4 x double> @llvm.x86.fma.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
declare <4 x double> @llvm.x86.fma.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
declare <4 x double> @llvm.x86.fma.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
declare <4 x double> @llvm.x86.fma.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)
declare <4 x double> @llvm.x86.fma.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>)
declare <4 x double> @llvm.x86.fma.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>)

define <8 x float> @fmaddsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fmaddsubps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB18_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB18_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmaddsub231ps {{.*#+}} ymm2 = (ymm0 * ymm1) +/- ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB18_2
; CHECK-NEXT:  .LBB18_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

define <8 x float> @fmsubaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fmsubaddps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB19_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB19_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsubadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) -/+ ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB19_2
; CHECK-NEXT:  .LBB19_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

define <8 x float> @fmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fmaddps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB20_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB20_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmadd231ps {{.*#+}} ymm2 = (ymm0 * ymm1) + ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB20_2
; CHECK-NEXT:  .LBB20_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

define <8 x float> @fmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fmsubps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB21_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB21_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfmsub231ps {{.*#+}} ymm2 = (ymm0 * ymm1) - ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB21_2
; CHECK-NEXT:  .LBB21_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

define <8 x float> @fnmaddps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fnmaddps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB22_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB22_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmadd231ps {{.*#+}} ymm2 = -(ymm0 * ymm1) + ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB22_2
; CHECK-NEXT:  .LBB22_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

define <8 x float> @fnmsubps_loop_256(i32 %iter, <8 x float> %a, <8 x float> %b, <8 x float> %c) {
; CHECK-LABEL: fnmsubps_loop_256:
; CHECK:       # %bb.0: # %entry
; CHECK-NEXT:    xorl %eax, %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jge .LBB23_3
; CHECK-NEXT:    .p2align 4, 0x90
; CHECK-NEXT:  .LBB23_2: # %for.body
; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    vfnmsub231ps {{.*#+}} ymm2 = -(ymm0 * ymm1) - ymm2
; CHECK-NEXT:    incl %eax
; CHECK-NEXT:    cmpl %edi, %eax
; CHECK-NEXT:    jl .LBB23_2
; CHECK-NEXT:  .LBB23_3: # %for.end
; CHECK-NEXT:    vmovaps %ymm2, %ymm0
; CHECK-NEXT:    retq
entry:
  br label %for.cond

for.cond:
  %c.addr.0 = phi <8 x float> [ %c, %entry ], [ %0, %for.inc ]
  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
  %cmp = icmp slt i32 %i.0, %iter
  br i1 %cmp, label %for.body, label %for.end

for.body:
  br label %for.inc

for.inc:
  %0 = call <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c.addr.0)
  %inc = add nsw i32 %i.0, 1
  br label %for.cond

for.end:
  ret <8 x float> %c.addr.0
}

declare <8 x float> @llvm.x86.fma.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
declare <8 x float> @llvm.x86.fma.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
declare <8 x float> @llvm.x86.fma.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
declare <8 x float> @llvm.x86.fma.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)
declare <8 x float> @llvm.x86.fma.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>)
declare <8 x float> @llvm.x86.fma.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>)