1; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -march=amdgcn -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s 2; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=1 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVGPR -check-prefix=GCN %s 3; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -march=amdgcn -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s 4; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=0 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOVMEM -check-prefix=GCN %s 5; RUN: llc -O0 -amdgpu-spill-sgpr-to-vgpr=0 -amdgpu-spill-sgpr-to-smem=1 -march=amdgcn -mcpu=tonga -mattr=+vgpr-spilling -verify-machineinstrs < %s | FileCheck -check-prefix=TOSMEM -check-prefix=GCN %s 6 7; XXX - Why does it like to use vcc? 8 9; GCN-LABEL: {{^}}spill_m0: 10; TOSMEM: s_mov_b32 s[[LO:[0-9]+]], SCRATCH_RSRC_DWORD0 11; TOSMEM: s_mov_b32 s[[HI:[0-9]+]], 0xe80000 12 13; GCN-DAG: s_cmp_lg_u32 14 15; TOVGPR-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 16; TOVGPR: v_writelane_b32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]], 0 17 18; TOVMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 19; TOVMEM-DAG: v_mov_b32_e32 [[SPILL_VREG:v[0-9]+]], [[M0_COPY]] 20; TOVMEM: buffer_store_dword [[SPILL_VREG]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4 ; 4-byte Folded Spill 21 22; TOSMEM-DAG: s_mov_b32 [[M0_COPY:s[0-9]+]], m0 23; TOSMEM: s_add_u32 m0, s3, 0x100{{$}} 24; TOSMEM-NOT: [[M0_COPY]] 25; TOSMEM: s_buffer_store_dword [[M0_COPY]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Spill 26 27; GCN: s_cbranch_scc1 [[ENDIF:BB[0-9]+_[0-9]+]] 28 29; GCN: [[ENDIF]]: 30; TOVGPR: v_readlane_b32 [[M0_RESTORE:s[0-9]+]], [[SPILL_VREG]], 0 31; TOVGPR: s_mov_b32 m0, [[M0_RESTORE]] 32 33; TOVMEM: buffer_load_dword [[RELOAD_VREG:v[0-9]+]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offset:4 ; 4-byte Folded Reload 34; TOVMEM: s_waitcnt vmcnt(0) 35; TOVMEM: v_readfirstlane_b32 [[M0_RESTORE:s[0-9]+]], [[RELOAD_VREG]] 36; TOVMEM: s_mov_b32 m0, [[M0_RESTORE]] 37 38; TOSMEM: s_add_u32 m0, s3, 0x100{{$}} 39; TOSMEM: s_buffer_load_dword [[M0_RESTORE:s[0-9]+]], s{{\[}}[[LO]]:[[HI]]], m0 ; 4-byte Folded Reload 40; TOSMEM-NOT: [[M0_RESTORE]] 41; TOSMEM: s_mov_b32 m0, [[M0_RESTORE]] 42 43; GCN: s_add_i32 s{{[0-9]+}}, m0, 1 44define amdgpu_kernel void @spill_m0(i32 %cond, i32 addrspace(1)* %out) #0 { 45entry: 46 %m0 = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0 47 %cmp0 = icmp eq i32 %cond, 0 48 br i1 %cmp0, label %if, label %endif 49 50if: 51 call void asm sideeffect "v_nop", ""() #0 52 br label %endif 53 54endif: 55 %foo = call i32 asm sideeffect "s_add_i32 $0, $1, 1", "=s,{M0}"(i32 %m0) #0 56 store i32 %foo, i32 addrspace(1)* %out 57 ret void 58} 59 60@lds = internal addrspace(3) global [64 x float] undef 61 62; m0 is killed, so it isn't necessary during the entry block spill to preserve it 63; GCN-LABEL: {{^}}spill_kill_m0_lds: 64; GCN: s_mov_b32 m0, s6 65; GCN: v_interp_mov_f32 66 67; TOSMEM-NOT: s_m0 68; TOSMEM: s_add_u32 m0, s7, 0x100 69; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill 70; FIXME: RegScavenger::isRegUsed() always returns true if m0 is reserved, so we have to save and restore it 71; FIXME-TOSMEM-NOT: m0 72 73; FIXME-TOSMEM-NOT: m0 74; TOSMEM: s_add_u32 m0, s7, 0x300 75; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 4-byte Folded Spill 76; FIXME-TOSMEM-NOT: m0 77 78; TOSMEM: s_mov_b64 exec, 79; TOSMEM: s_cbranch_execz 80; TOSMEM: s_branch 81 82; TOSMEM: BB{{[0-9]+_[0-9]+}}: 83; TOSMEM: s_add_u32 m0, s7, 0x400 84; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload 85 86 87; GCN-NOT: v_readlane_b32 m0 88; GCN-NOT: s_buffer_store_dword m0 89; GCN-NOT: s_buffer_load_dword m0 90define amdgpu_ps void @spill_kill_m0_lds(<16 x i8> addrspace(4)* inreg %arg, <16 x i8> addrspace(4)* inreg %arg1, <32 x i8> addrspace(4)* inreg %arg2, i32 inreg %m0) #0 { 91main_body: 92 %tmp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0) 93 %cmp = fcmp ueq float 0.000000e+00, %tmp 94 br i1 %cmp, label %if, label %else 95 96if: ; preds = %main_body 97 %lds_ptr = getelementptr [64 x float], [64 x float] addrspace(3)* @lds, i32 0, i32 0 98 %lds_data_ = load float, float addrspace(3)* %lds_ptr 99 %lds_data = call float @llvm.amdgcn.wqm.f32(float %lds_data_) 100 br label %endif 101 102else: ; preds = %main_body 103 %interp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0) 104 br label %endif 105 106endif: ; preds = %else, %if 107 %export = phi float [ %lds_data, %if ], [ %interp, %else ] 108 %tmp4 = call <2 x half> @llvm.amdgcn.cvt.pkrtz(float %export, float %export) 109 call void @llvm.amdgcn.exp.compr.v2f16(i32 0, i32 15, <2 x half> %tmp4, <2 x half> %tmp4, i1 true, i1 true) #0 110 ret void 111} 112 113; Force save and restore of m0 during SMEM spill 114; GCN-LABEL: {{^}}m0_unavailable_spill: 115 116; GCN: ; def m0, 1 117 118; GCN: s_mov_b32 m0, s2 119; GCN: v_interp_mov_f32 120 121; GCN: ; clobber m0 122 123; TOSMEM: s_mov_b32 s2, m0 124; TOSMEM: s_add_u32 m0, s3, 0x100 125; TOSMEM-NEXT: s_buffer_store_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Spill 126; TOSMEM: s_mov_b32 m0, s2 127 128; TOSMEM: s_mov_b64 exec, 129; TOSMEM: s_cbranch_execz 130; TOSMEM: s_branch 131 132; TOSMEM: BB{{[0-9]+_[0-9]+}}: 133; TOSMEM: s_add_u32 m0, s3, 0x100 134; TOSMEM-NEXT: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, m0 ; 8-byte Folded Reload 135 136; GCN-NOT: v_readlane_b32 m0 137; GCN-NOT: s_buffer_store_dword m0 138; GCN-NOT: s_buffer_load_dword m0 139define amdgpu_kernel void @m0_unavailable_spill(i32 %m0.arg) #0 { 140main_body: 141 %m0 = call i32 asm sideeffect "; def $0, 1", "={M0}"() #0 142 %tmp = call float @llvm.amdgcn.interp.mov(i32 2, i32 0, i32 0, i32 %m0.arg) 143 call void asm sideeffect "; clobber $0", "~{M0}"() #0 144 %cmp = fcmp ueq float 0.000000e+00, %tmp 145 br i1 %cmp, label %if, label %else 146 147if: ; preds = %main_body 148 store volatile i32 8, i32 addrspace(1)* undef 149 br label %endif 150 151else: ; preds = %main_body 152 store volatile i32 11, i32 addrspace(1)* undef 153 br label %endif 154 155endif: 156 ret void 157} 158 159; GCN-LABEL: {{^}}restore_m0_lds: 160; TOSMEM: s_load_dwordx2 [[REG:s\[[0-9]+:[0-9]+\]]] 161; TOSMEM: s_cmp_eq_u32 162; FIXME: RegScavenger::isRegUsed() always returns true if m0 is reserved, so we have to save and restore it 163; FIXME-TOSMEM-NOT: m0 164; TOSMEM: s_add_u32 m0, s3, 0x100 165; TOSMEM: s_buffer_store_dwordx2 [[REG]], s[88:91], m0 ; 8-byte Folded Spill 166; FIXME-TOSMEM-NOT: m0 167; TOSMEM: s_add_u32 m0, s3, 0x300 168; TOSMEM: s_buffer_store_dword s{{[0-9]+}}, s[88:91], m0 ; 4-byte Folded Spill 169; FIXME-TOSMEM-NOT: m0 170; TOSMEM: s_cbranch_scc1 171 172; TOSMEM: s_mov_b32 m0, -1 173 174; TOSMEM: s_mov_b32 s0, m0 175; TOSMEM: s_add_u32 m0, s3, 0x100 176; TOSMEM: s_buffer_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s[88:91], m0 ; 8-byte Folded Reload 177; TOSMEM: s_mov_b32 m0, s0 178; TOSMEM: s_waitcnt lgkmcnt(0) 179 180; TOSMEM: ds_write_b64 181 182; FIXME-TOSMEM-NOT: m0 183; TOSMEM: s_add_u32 m0, s3, 0x300 184; TOSMEM: s_buffer_load_dword s0, s[88:91], m0 ; 4-byte Folded Reload 185; FIXME-TOSMEM-NOT: m0 186; TOSMEM: s_waitcnt lgkmcnt(0) 187; TOSMEM-NOT: m0 188; TOSMEM: s_mov_b32 m0, s0 189; TOSMEM: ; use m0 190 191; TOSMEM: s_dcache_wb 192; TOSMEM: s_endpgm 193define amdgpu_kernel void @restore_m0_lds(i32 %arg) { 194 %m0 = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0 195 %sval = load volatile i64, i64 addrspace(4)* undef 196 %cmp = icmp eq i32 %arg, 0 197 br i1 %cmp, label %ret, label %bb 198 199bb: 200 store volatile i64 %sval, i64 addrspace(3)* undef 201 call void asm sideeffect "; use $0", "{M0}"(i32 %m0) #0 202 br label %ret 203 204ret: 205 ret void 206} 207 208declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #1 209declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0 210declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #0 211declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #1 212declare float @llvm.amdgcn.wqm.f32(float) #1 213 214attributes #0 = { nounwind } 215attributes #1 = { nounwind readnone } 216