1; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs -mattr=+load-store-opt < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,CI %s 2; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -mattr=+load-store-opt,+flat-for-global < %s | FileCheck -enable-var-scope -strict-whitespace -check-prefixes=GCN,GFX9 %s 3 4; FIXME: We don't get cases where the address was an SGPR because we 5; get a copy to the address register for each one. 6 7@lds = addrspace(3) global [512 x float] undef, align 4 8@lds.f64 = addrspace(3) global [512 x double] undef, align 8 9 10; GCN-LABEL: {{^}}simple_read2_f32: 11; CI-DAG: s_mov_b32 m0 12; GFX9-NOT: m0 13 14; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:8 15; GCN: s_waitcnt lgkmcnt(0) 16; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 17; CI: buffer_store_dword [[RESULT]] 18; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 19; GCN: s_endpgm 20define amdgpu_kernel void @simple_read2_f32(float addrspace(1)* %out) #0 { 21 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 22 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 23 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 24 %add.x = add nsw i32 %x.i, 8 25 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 26 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 27 %sum = fadd float %val0, %val1 28 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 29 store float %sum, float addrspace(1)* %out.gep, align 4 30 ret void 31} 32 33; GCN-LABEL: {{^}}simple_read2_f32_max_offset: 34; CI-DAG: s_mov_b32 m0 35; GFX9-NOT: m0 36 37; GCN: ds_read2_b32 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, v{{[0-9]+}} offset1:255 38; GCN: s_waitcnt lgkmcnt(0) 39; GCN: v_add_f32_e32 [[RESULT:v[0-9]+]], v[[LO_VREG]], v[[HI_VREG]] 40 41; CI: buffer_store_dword [[RESULT]] 42; GFX9: global_store_dword v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 43define amdgpu_kernel void @simple_read2_f32_max_offset(float addrspace(1)* %out) #0 { 44 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 45 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 46 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 47 %add.x = add nsw i32 %x.i, 255 48 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 49 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 50 %sum = fadd float %val0, %val1 51 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 52 store float %sum, float addrspace(1)* %out.gep, align 4 53 ret void 54} 55 56; GCN-LABEL: @simple_read2_f32_too_far 57; CI-DAG: s_mov_b32 m0 58; GFX9-NOT: m0 59 60; GCN-NOT ds_read2_b32 61; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 62; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:1028 63; GCN: s_endpgm 64define amdgpu_kernel void @simple_read2_f32_too_far(float addrspace(1)* %out) #0 { 65 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 66 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 67 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 68 %add.x = add nsw i32 %x.i, 257 69 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 70 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 71 %sum = fadd float %val0, %val1 72 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 73 store float %sum, float addrspace(1)* %out.gep, align 4 74 ret void 75} 76 77; GCN-LABEL: @simple_read2_f32_x2 78; CI-DAG: s_mov_b32 m0 79; GFX9-NOT: m0 80 81; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 82; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 83; GCN: s_endpgm 84define amdgpu_kernel void @simple_read2_f32_x2(float addrspace(1)* %out) #0 { 85 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 86 %idx.0 = add nsw i32 %tid.x, 0 87 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 88 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 89 90 %idx.1 = add nsw i32 %tid.x, 8 91 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 92 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 93 %sum.0 = fadd float %val0, %val1 94 95 %idx.2 = add nsw i32 %tid.x, 11 96 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 97 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 98 99 %idx.3 = add nsw i32 %tid.x, 27 100 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 101 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 102 %sum.1 = fadd float %val2, %val3 103 104 %sum = fadd float %sum.0, %sum.1 105 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 106 store float %sum, float addrspace(1)* %out.gep, align 4 107 ret void 108} 109 110; Make sure there is an instruction between the two sets of reads. 111; GCN-LABEL: @simple_read2_f32_x2_barrier 112; CI-DAG: s_mov_b32 m0 113; GFX9-NOT: m0 114 115; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset1:8 116; GCN: s_barrier 117; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 118; GCN: s_endpgm 119define amdgpu_kernel void @simple_read2_f32_x2_barrier(float addrspace(1)* %out) #0 { 120 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 121 %idx.0 = add nsw i32 %tid.x, 0 122 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 123 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 124 125 %idx.1 = add nsw i32 %tid.x, 8 126 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 127 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 128 %sum.0 = fadd float %val0, %val1 129 130 call void @llvm.amdgcn.s.barrier() #2 131 132 %idx.2 = add nsw i32 %tid.x, 11 133 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 134 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 135 136 %idx.3 = add nsw i32 %tid.x, 27 137 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 138 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 139 %sum.1 = fadd float %val2, %val3 140 141 %sum = fadd float %sum.0, %sum.1 142 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 143 store float %sum, float addrspace(1)* %out.gep, align 4 144 ret void 145} 146 147; For some reason adding something to the base address for the first 148; element results in only folding the inner pair. 149 150; GCN-LABEL: @simple_read2_f32_x2_nonzero_base 151; CI-DAG: s_mov_b32 m0 152; GFX9-NOT: m0 153 154; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR:v[0-9]+]] offset0:2 offset1:8 155; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASEADDR]] offset0:11 offset1:27 156; GCN: s_endpgm 157define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(float addrspace(1)* %out) #0 { 158 %tid.x = tail call i32 @llvm.amdgcn.workitem.id.x() #1 159 %idx.0 = add nsw i32 %tid.x, 2 160 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.0 161 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 162 163 %idx.1 = add nsw i32 %tid.x, 8 164 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.1 165 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 166 %sum.0 = fadd float %val0, %val1 167 168 %idx.2 = add nsw i32 %tid.x, 11 169 %arrayidx2 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.2 170 %val2 = load float, float addrspace(3)* %arrayidx2, align 4 171 172 %idx.3 = add nsw i32 %tid.x, 27 173 %arrayidx3 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %idx.3 174 %val3 = load float, float addrspace(3)* %arrayidx3, align 4 175 %sum.1 = fadd float %val2, %val3 176 177 %sum = fadd float %sum.0, %sum.1 178 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %idx.0 179 store float %sum, float addrspace(1)* %out.gep, align 4 180 ret void 181} 182 183; Be careful of vectors of pointers. We don't know if the 2 pointers 184; in the vectors are really the same base, so this is not safe to 185; merge. 186; Base pointers come from different subregister of same super 187; register. We can't safely merge this. 188 189; GCN-LABEL: @read2_ptr_is_subreg_arg_f32 190; CI-DAG: s_mov_b32 m0 191; GFX9-NOT: m0 192 193; GCN-NOT: ds_read2_b32 194; GCN: ds_read_b32 195; GCN: ds_read_b32 196; GCN: s_endpgm 197define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 198 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 199 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 200 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 201 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 202 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 203 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 204 %val0 = load float, float addrspace(3)* %gep.0, align 4 205 %val1 = load float, float addrspace(3)* %gep.1, align 4 206 %add.x = add nsw i32 %x.i, 8 207 %sum = fadd float %val0, %val1 208 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 209 store float %sum, float addrspace(1)* %out.gep, align 4 210 ret void 211} 212 213; Apply a constant scalar offset after the pointer vector extract. We 214; are rejecting merges that have the same, constant 0 offset, so make 215; sure we are really rejecting it because of the different 216; subregisters. 217 218; GCN-LABEL: @read2_ptr_is_subreg_arg_offset_f32 219; CI-DAG: s_mov_b32 m0 220; GFX9-NOT: m0 221 222; GCN-NOT: ds_read2_b32 223; GCN: ds_read_b32 224; GCN: ds_read_b32 225; GCN: s_endpgm 226define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(float addrspace(1)* %out, <2 x float addrspace(3)*> %lds.ptr) #0 { 227 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 228 %index.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 229 %index.1 = insertelement <2 x i32> %index.0, i32 8, i32 0 230 %gep = getelementptr inbounds float, <2 x float addrspace(3)*> %lds.ptr, <2 x i32> %index.1 231 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 232 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 233 234 ; Apply an additional offset after the vector that will be more obviously folded. 235 %gep.1.offset = getelementptr float, float addrspace(3)* %gep.1, i32 8 236 237 %val0 = load float, float addrspace(3)* %gep.0, align 4 238 %val1 = load float, float addrspace(3)* %gep.1.offset, align 4 239 %add.x = add nsw i32 %x.i, 8 240 %sum = fadd float %val0, %val1 241 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 242 store float %sum, float addrspace(1)* %out.gep, align 4 243 ret void 244} 245 246; GCN-LABEL: {{^}}read2_ptr_is_subreg_f32: 247; CI-DAG: s_mov_b32 m0 248; GFX9-NOT: m0 249 250; GCN: ds_read2_b32 {{v\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:8{{$}} 251; GCN: s_endpgm 252define amdgpu_kernel void @read2_ptr_is_subreg_f32(float addrspace(1)* %out) #0 { 253 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 254 %ptr.0 = insertelement <2 x [512 x float] addrspace(3)*> undef, [512 x float] addrspace(3)* @lds, i32 0 255 %ptr.1 = insertelement <2 x [512 x float] addrspace(3)*> %ptr.0, [512 x float] addrspace(3)* @lds, i32 1 256 %x.i.v.0 = insertelement <2 x i32> undef, i32 %x.i, i32 0 257 %x.i.v.1 = insertelement <2 x i32> %x.i.v.0, i32 %x.i, i32 1 258 %idx = add <2 x i32> %x.i.v.1, <i32 0, i32 8> 259 %gep = getelementptr inbounds [512 x float], <2 x [512 x float] addrspace(3)*> %ptr.1, <2 x i32> <i32 0, i32 0>, <2 x i32> %idx 260 %gep.0 = extractelement <2 x float addrspace(3)*> %gep, i32 0 261 %gep.1 = extractelement <2 x float addrspace(3)*> %gep, i32 1 262 %val0 = load float, float addrspace(3)* %gep.0, align 4 263 %val1 = load float, float addrspace(3)* %gep.1, align 4 264 %add.x = add nsw i32 %x.i, 8 265 %sum = fadd float %val0, %val1 266 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 267 store float %sum, float addrspace(1)* %out.gep, align 4 268 ret void 269} 270 271; GCN-LABEL: @simple_read2_f32_volatile_0 272; CI-DAG: s_mov_b32 m0 273; GFX9-NOT: m0 274 275; GCN-NOT ds_read2_b32 276; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 277; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 278; GCN: s_endpgm 279define amdgpu_kernel void @simple_read2_f32_volatile_0(float addrspace(1)* %out) #0 { 280 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 281 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 282 %val0 = load volatile float, float addrspace(3)* %arrayidx0, align 4 283 %add.x = add nsw i32 %x.i, 8 284 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 285 %val1 = load float, float addrspace(3)* %arrayidx1, align 4 286 %sum = fadd float %val0, %val1 287 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 288 store float %sum, float addrspace(1)* %out.gep, align 4 289 ret void 290} 291 292; GCN-LABEL: @simple_read2_f32_volatile_1 293; CI-DAG: s_mov_b32 m0 294; GFX9-NOT: m0 295 296; GCN-NOT ds_read2_b32 297; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} 298; GCN: ds_read_b32 v{{[0-9]+}}, v{{[0-9]+}} offset:32 299; GCN: s_endpgm 300define amdgpu_kernel void @simple_read2_f32_volatile_1(float addrspace(1)* %out) #0 { 301 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 302 %arrayidx0 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %x.i 303 %val0 = load float, float addrspace(3)* %arrayidx0, align 4 304 %add.x = add nsw i32 %x.i, 8 305 %arrayidx1 = getelementptr inbounds [512 x float], [512 x float] addrspace(3)* @lds, i32 0, i32 %add.x 306 %val1 = load volatile float, float addrspace(3)* %arrayidx1, align 4 307 %sum = fadd float %val0, %val1 308 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 309 store float %sum, float addrspace(1)* %out.gep, align 4 310 ret void 311} 312 313; Can't fold since not correctly aligned. 314; XXX: This isn't really testing anything useful now. I think CI 315; allows unaligned LDS accesses, which would be a problem here. 316; GCN-LABEL: @unaligned_read2_f32 317; CI-DAG: s_mov_b32 m0 318; GFX9-NOT: m0 319 320; GCN-NOT: ds_read2_b32 321; GCN: s_endpgm 322define amdgpu_kernel void @unaligned_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 323 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 324 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 325 %val0 = load float, float addrspace(3)* %arrayidx0, align 1 326 %add.x = add nsw i32 %x.i, 8 327 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 328 %val1 = load float, float addrspace(3)* %arrayidx1, align 1 329 %sum = fadd float %val0, %val1 330 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 331 store float %sum, float addrspace(1)* %out.gep, align 4 332 ret void 333} 334 335; GCN-LABEL: @misaligned_2_simple_read2_f32 336; CI-DAG: s_mov_b32 m0 337; GFX9-NOT: m0 338 339; GCN-NOT: ds_read2_b32 340; GCN: s_endpgm 341define amdgpu_kernel void @misaligned_2_simple_read2_f32(float addrspace(1)* %out, float addrspace(3)* %lds) #0 { 342 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 343 %arrayidx0 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %x.i 344 %val0 = load float, float addrspace(3)* %arrayidx0, align 2 345 %add.x = add nsw i32 %x.i, 8 346 %arrayidx1 = getelementptr inbounds float, float addrspace(3)* %lds, i32 %add.x 347 %val1 = load float, float addrspace(3)* %arrayidx1, align 2 348 %sum = fadd float %val0, %val1 349 %out.gep = getelementptr inbounds float, float addrspace(1)* %out, i32 %x.i 350 store float %sum, float addrspace(1)* %out.gep, align 4 351 ret void 352} 353 354; GCN-LABEL: @simple_read2_f64 355; CI-DAG: s_mov_b32 m0 356; GFX9-NOT: m0 357 358; GCN-DAG: v_lshlrev_b32_e32 [[VPTR:v[0-9]+]], 3, {{v[0-9]+}} 359; GCN: ds_read2_b64 v{{\[}}[[LO_VREG:[0-9]+]]:[[HI_VREG:[0-9]+]]{{\]}}, [[VPTR]] offset1:8 360; GCN: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], v{{\[}}[[LO_VREG]]:{{[0-9]+\]}}, v{{\[[0-9]+}}:[[HI_VREG]]{{\]}} 361 362; CI: buffer_store_dwordx2 [[RESULT]] 363; GFX9: global_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RESULT]] 364define amdgpu_kernel void @simple_read2_f64(double addrspace(1)* %out) #0 { 365 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 366 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 367 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 368 %add.x = add nsw i32 %x.i, 8 369 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 370 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 371 %sum = fadd double %val0, %val1 372 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 373 store double %sum, double addrspace(1)* %out.gep, align 8 374 ret void 375} 376 377; GCN-LABEL: @simple_read2_f64_max_offset 378; CI-DAG: s_mov_b32 m0 379; GFX9-NOT: m0 380 381; GCN: ds_read2_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset1:255 382; GCN: s_endpgm 383define amdgpu_kernel void @simple_read2_f64_max_offset(double addrspace(1)* %out) #0 { 384 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 385 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 386 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 387 %add.x = add nsw i32 %x.i, 255 388 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 389 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 390 %sum = fadd double %val0, %val1 391 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 392 store double %sum, double addrspace(1)* %out.gep, align 8 393 ret void 394} 395 396; GCN-LABEL: @simple_read2_f64_too_far 397; CI-DAG: s_mov_b32 m0 398; GFX9-NOT: m0 399 400; GCN-NOT ds_read2_b64 401; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} 402; GCN: ds_read_b64 {{v\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset:2056 403; GCN: s_endpgm 404define amdgpu_kernel void @simple_read2_f64_too_far(double addrspace(1)* %out) #0 { 405 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 406 %arrayidx0 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %x.i 407 %val0 = load double, double addrspace(3)* %arrayidx0, align 8 408 %add.x = add nsw i32 %x.i, 257 409 %arrayidx1 = getelementptr inbounds [512 x double], [512 x double] addrspace(3)* @lds.f64, i32 0, i32 %add.x 410 %val1 = load double, double addrspace(3)* %arrayidx1, align 8 411 %sum = fadd double %val0, %val1 412 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 413 store double %sum, double addrspace(1)* %out.gep, align 8 414 ret void 415} 416 417; Alignment only 4 418; GCN-LABEL: @misaligned_read2_f64 419; CI-DAG: s_mov_b32 m0 420; GFX9-NOT: m0 421 422; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset1:1 423; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, {{v[0-9]+}} offset0:14 offset1:15 424; GCN: s_endpgm 425define amdgpu_kernel void @misaligned_read2_f64(double addrspace(1)* %out, double addrspace(3)* %lds) #0 { 426 %x.i = tail call i32 @llvm.amdgcn.workitem.id.x() #1 427 %arrayidx0 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %x.i 428 %val0 = load double, double addrspace(3)* %arrayidx0, align 4 429 %add.x = add nsw i32 %x.i, 7 430 %arrayidx1 = getelementptr inbounds double, double addrspace(3)* %lds, i32 %add.x 431 %val1 = load double, double addrspace(3)* %arrayidx1, align 4 432 %sum = fadd double %val0, %val1 433 %out.gep = getelementptr inbounds double, double addrspace(1)* %out, i32 %x.i 434 store double %sum, double addrspace(1)* %out.gep, align 4 435 ret void 436} 437 438@foo = addrspace(3) global [4 x i32] undef, align 4 439 440; GCN-LABEL: @load_constant_adjacent_offsets 441; CI-DAG: s_mov_b32 m0 442; GFX9-NOT: m0 443 444; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 445; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1 446define amdgpu_kernel void @load_constant_adjacent_offsets(i32 addrspace(1)* %out) { 447 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 448 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 1), align 4 449 %sum = add i32 %val0, %val1 450 store i32 %sum, i32 addrspace(1)* %out, align 4 451 ret void 452} 453 454; GCN-LABEL: @load_constant_disjoint_offsets 455; CI-DAG: s_mov_b32 m0 456; GFX9-NOT: m0 457 458; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 459; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:2 460define amdgpu_kernel void @load_constant_disjoint_offsets(i32 addrspace(1)* %out) { 461 %val0 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 0), align 4 462 %val1 = load i32, i32 addrspace(3)* getelementptr inbounds ([4 x i32], [4 x i32] addrspace(3)* @foo, i32 0, i32 2), align 4 463 %sum = add i32 %val0, %val1 464 store i32 %sum, i32 addrspace(1)* %out, align 4 465 ret void 466} 467 468@bar = addrspace(3) global [4 x i64] undef, align 4 469 470; GCN-LABEL: @load_misaligned64_constant_offsets 471; CI-DAG: s_mov_b32 m0 472; GFX9-NOT: m0 473 474; GCN-DAG: v_mov_b32_e32 [[ZERO:v[0-9]+]], 0{{$}} 475; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset1:1 476; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[ZERO]] offset0:2 offset1:3 477define amdgpu_kernel void @load_misaligned64_constant_offsets(i64 addrspace(1)* %out) { 478 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 0), align 4 479 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4 x i64], [4 x i64] addrspace(3)* @bar, i32 0, i32 1), align 4 480 %sum = add i64 %val0, %val1 481 store i64 %sum, i64 addrspace(1)* %out, align 8 482 ret void 483} 484 485@bar.large = addrspace(3) global [4096 x i64] undef, align 4 486 487; GCN-LABEL: @load_misaligned64_constant_large_offsets 488; CI-DAG: s_mov_b32 m0 489; GFX9-NOT: m0 490 491; GCN-DAG: v_mov_b32_e32 [[BASE0:v[0-9]+]], 0x7ff8{{$}} 492; GCN-DAG: v_mov_b32_e32 [[BASE1:v[0-9]+]], 0x4000 493; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE0]] offset1:1 494; GCN-DAG: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, [[BASE1]] offset1:1 495; GCN: s_endpgm 496define amdgpu_kernel void @load_misaligned64_constant_large_offsets(i64 addrspace(1)* %out) { 497 %val0 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 2048), align 4 498 %val1 = load i64, i64 addrspace(3)* getelementptr inbounds ([4096 x i64], [4096 x i64] addrspace(3)* @bar.large, i32 0, i32 4095), align 4 499 %sum = add i64 %val0, %val1 500 store i64 %sum, i64 addrspace(1)* %out, align 8 501 ret void 502} 503 504@sgemm.lA = internal unnamed_addr addrspace(3) global [264 x float] undef, align 4 505@sgemm.lB = internal unnamed_addr addrspace(3) global [776 x float] undef, align 4 506 507; GCN-LABEL: {{^}}sgemm_inner_loop_read2_sequence: 508; CI-DAG: s_mov_b32 m0 509; GFX9-NOT: m0 510 511define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(float addrspace(1)* %C, i32 %lda, i32 %ldb) #0 { 512 %x.i = tail call i32 @llvm.amdgcn.workgroup.id.x() #1 513 %y.i = tail call i32 @llvm.amdgcn.workitem.id.y() #1 514 %arrayidx44 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %x.i 515 %tmp16 = load float, float addrspace(3)* %arrayidx44, align 4 516 %add47 = add nsw i32 %x.i, 1 517 %arrayidx48 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add47 518 %tmp17 = load float, float addrspace(3)* %arrayidx48, align 4 519 %add51 = add nsw i32 %x.i, 16 520 %arrayidx52 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add51 521 %tmp18 = load float, float addrspace(3)* %arrayidx52, align 4 522 %add55 = add nsw i32 %x.i, 17 523 %arrayidx56 = getelementptr inbounds [264 x float], [264 x float] addrspace(3)* @sgemm.lA, i32 0, i32 %add55 524 %tmp19 = load float, float addrspace(3)* %arrayidx56, align 4 525 %arrayidx60 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %y.i 526 %tmp20 = load float, float addrspace(3)* %arrayidx60, align 4 527 %add63 = add nsw i32 %y.i, 1 528 %arrayidx64 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add63 529 %tmp21 = load float, float addrspace(3)* %arrayidx64, align 4 530 %add67 = add nsw i32 %y.i, 32 531 %arrayidx68 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add67 532 %tmp22 = load float, float addrspace(3)* %arrayidx68, align 4 533 %add71 = add nsw i32 %y.i, 33 534 %arrayidx72 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add71 535 %tmp23 = load float, float addrspace(3)* %arrayidx72, align 4 536 %add75 = add nsw i32 %y.i, 64 537 %arrayidx76 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add75 538 %tmp24 = load float, float addrspace(3)* %arrayidx76, align 4 539 %add79 = add nsw i32 %y.i, 65 540 %arrayidx80 = getelementptr inbounds [776 x float], [776 x float] addrspace(3)* @sgemm.lB, i32 0, i32 %add79 541 %tmp25 = load float, float addrspace(3)* %arrayidx80, align 4 542 %sum.0 = fadd float %tmp16, %tmp17 543 %sum.1 = fadd float %sum.0, %tmp18 544 %sum.2 = fadd float %sum.1, %tmp19 545 %sum.3 = fadd float %sum.2, %tmp20 546 %sum.4 = fadd float %sum.3, %tmp21 547 %sum.5 = fadd float %sum.4, %tmp22 548 %sum.6 = fadd float %sum.5, %tmp23 549 %sum.7 = fadd float %sum.6, %tmp24 550 %sum.8 = fadd float %sum.7, %tmp25 551 store float %sum.8, float addrspace(1)* %C, align 4 552 ret void 553} 554 555; GCN-LABEL: {{^}}misaligned_read2_v2i32: 556; CI-DAG: s_mov_b32 m0 557; GFX9-NOT: m0 558define amdgpu_kernel void @misaligned_read2_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(3)* %in) #0 { 559 %load = load <2 x i32>, <2 x i32> addrspace(3)* %in, align 4 560 store <2 x i32> %load, <2 x i32> addrspace(1)* %out, align 8 561 ret void 562} 563 564; GCN-LABEL: {{^}}misaligned_read2_i64: 565; CI-DAG: s_mov_b32 m0 566; GFX9-NOT: m0 567define amdgpu_kernel void @misaligned_read2_i64(i64 addrspace(1)* %out, i64 addrspace(3)* %in) #0 { 568 %load = load i64, i64 addrspace(3)* %in, align 4 569 store i64 %load, i64 addrspace(1)* %out, align 8 570 ret void 571} 572 573; GCN-LABEL: ds_read_diff_base_interleaving 574; CI-DAG: s_mov_b32 m0 575; GFX9-NOT: m0 576 577; GCN-NOT: ds_read_b32 578define amdgpu_kernel void @ds_read_diff_base_interleaving( 579 float addrspace(1)* nocapture %arg, 580 [4 x [4 x float]] addrspace(3)* %arg1, 581 [4 x [4 x float]] addrspace(3)* %arg2, 582 [4 x [4 x float]] addrspace(3)* %arg3, 583 [4 x [4 x float]] addrspace(3)* %arg4) #1 { 584bb: 585 %tmp = getelementptr float, float addrspace(1)* %arg, i64 10 586 %tmp5 = tail call i32 @llvm.amdgcn.workitem.id.x() #2 587 %tmp6 = tail call i32 @llvm.amdgcn.workitem.id.y() #2 588 %tmp7 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 0 589 %tmp8 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 0, i32 %tmp5 590 %tmp9 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 0 591 %tmp10 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 0, i32 %tmp5 592 %tmp11 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg1, i32 0, i32 %tmp6, i32 1 593 %tmp12 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg2, i32 0, i32 1, i32 %tmp5 594 %tmp13 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg3, i32 0, i32 %tmp6, i32 1 595 %tmp14 = getelementptr [4 x [4 x float]], [4 x [4 x float]] addrspace(3)* %arg4, i32 0, i32 1, i32 %tmp5 596 %tmp15 = load float, float addrspace(3)* %tmp7 597 %tmp16 = load float, float addrspace(3)* %tmp8 598 %tmp17 = fmul float %tmp15, %tmp16 599 %tmp18 = fadd float 2.000000e+00, %tmp17 600 %tmp19 = load float, float addrspace(3)* %tmp9 601 %tmp20 = load float, float addrspace(3)* %tmp10 602 %tmp21 = fmul float %tmp19, %tmp20 603 %tmp22 = fsub float %tmp18, %tmp21 604 %tmp23 = load float, float addrspace(3)* %tmp11 605 %tmp24 = load float, float addrspace(3)* %tmp12 606 %tmp25 = fmul float %tmp23, %tmp24 607 %tmp26 = fsub float %tmp22, %tmp25 608 %tmp27 = load float, float addrspace(3)* %tmp13 609 %tmp28 = load float, float addrspace(3)* %tmp14 610 %tmp29 = fmul float %tmp27, %tmp28 611 %tmp30 = fsub float %tmp26, %tmp29 612 store float %tmp30, float addrspace(1)* %tmp 613 ret void 614} 615 616; GCN-LABEL: ds_read_call_read: 617; GCN: ds_read_b32 618; GCN: s_swappc_b64 619; GCN: ds_read_b32 620define amdgpu_kernel void @ds_read_call_read(i32 addrspace(1)* %out, i32 addrspace(3)* %arg) { 621 %x = call i32 @llvm.amdgcn.workitem.id.x() 622 %arrayidx0 = getelementptr i32, i32 addrspace(3)* %arg, i32 %x 623 %arrayidx1 = getelementptr i32, i32 addrspace(3)* %arrayidx0, i32 1 624 %v0 = load i32, i32 addrspace(3)* %arrayidx0, align 4 625 call void @void_func_void() 626 %v1 = load i32, i32 addrspace(3)* %arrayidx1, align 4 627 %r = add i32 %v0, %v1 628 store i32 %r, i32 addrspace(1)* %out, align 4 629 ret void 630} 631 632; GCN-LABEL: {{^}}ds_read_interp_read: 633; CI: s_mov_b32 m0, -1 634; CI: ds_read_b32 635; CI: s_mov_b32 m0, s0 636; CI: v_interp_mov_f32 637; CI: s_mov_b32 m0, -1 638; CI: ds_read_b32 639; GFX9: ds_read2_b32 v[0:1], v0 offset1:4 640; GFX9: s_mov_b32 m0, s0 641; GFX9: v_interp_mov_f32 642define amdgpu_ps <2 x float> @ds_read_interp_read(i32 inreg %prims, float addrspace(3)* %inptr) { 643 %v0 = load float, float addrspace(3)* %inptr, align 4 644 %intrp = call float @llvm.amdgcn.interp.mov(i32 0, i32 0, i32 0, i32 %prims) 645 %ptr1 = getelementptr float, float addrspace(3)* %inptr, i32 4 646 %v1 = load float, float addrspace(3)* %ptr1, align 4 647 %v1b = fadd float %v1, %intrp 648 %r0 = insertelement <2 x float> undef, float %v0, i32 0 649 %r1 = insertelement <2 x float> %r0, float %v1b, i32 1 650 ret <2 x float> %r1 651} 652 653declare void @void_func_void() #3 654 655declare i32 @llvm.amdgcn.workgroup.id.x() #1 656declare i32 @llvm.amdgcn.workgroup.id.y() #1 657declare i32 @llvm.amdgcn.workitem.id.x() #1 658declare i32 @llvm.amdgcn.workitem.id.y() #1 659 660declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) nounwind readnone 661 662declare void @llvm.amdgcn.s.barrier() #2 663 664attributes #0 = { nounwind } 665attributes #1 = { nounwind readnone speculatable } 666attributes #2 = { convergent nounwind } 667attributes #3 = { nounwind noinline } 668