1//===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9 10def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">; 11def MUBUFAddr64 : ComplexPattern<i64, 7, "SelectMUBUFAddr64">; 12def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">; 13 14def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>; 15def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>; 16 17def MUBUFOffset : ComplexPattern<i64, 6, "SelectMUBUFOffset">; 18def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">; 19def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">; 20def MUBUFIntrinsicOffset : ComplexPattern<i32, 2, "SelectMUBUFIntrinsicOffset">; 21def MUBUFIntrinsicVOffset : ComplexPattern<i32, 3, "SelectMUBUFIntrinsicVOffset">; 22 23class MubufLoad <SDPatternOperator op> : PatFrag < 24 (ops node:$ptr), (op node:$ptr), [{ 25 auto const AS = cast<MemSDNode>(N)->getAddressSpace(); 26 return AS == AMDGPUASI.GLOBAL_ADDRESS || 27 AS == AMDGPUASI.CONSTANT_ADDRESS; 28}]>; 29 30def mubuf_load : MubufLoad <load>; 31def mubuf_az_extloadi8 : MubufLoad <az_extloadi8>; 32def mubuf_sextloadi8 : MubufLoad <sextloadi8>; 33def mubuf_az_extloadi16 : MubufLoad <az_extloadi16>; 34def mubuf_sextloadi16 : MubufLoad <sextloadi16>; 35def mubuf_load_atomic : MubufLoad <atomic_load>; 36 37def BUFAddrKind { 38 int Offset = 0; 39 int OffEn = 1; 40 int IdxEn = 2; 41 int BothEn = 3; 42 int Addr64 = 4; 43} 44 45class getAddrName<int addrKind> { 46 string ret = 47 !if(!eq(addrKind, BUFAddrKind.Offset), "offset", 48 !if(!eq(addrKind, BUFAddrKind.OffEn), "offen", 49 !if(!eq(addrKind, BUFAddrKind.IdxEn), "idxen", 50 !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen", 51 !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64", 52 ""))))); 53} 54 55class MUBUFAddr64Table <bit is_addr64, string Name> { 56 bit IsAddr64 = is_addr64; 57 string OpName = Name; 58} 59 60class MUBUFLdsTable <bit is_lds, string Name> { 61 bit IsLds = is_lds; 62 string OpName = Name; 63} 64 65class MTBUFAddr64Table <bit is_addr64, string Name> { 66 bit IsAddr64 = is_addr64; 67 string OpName = Name; 68} 69 70//===----------------------------------------------------------------------===// 71// MTBUF classes 72//===----------------------------------------------------------------------===// 73 74class MTBUF_Pseudo <string opName, dag outs, dag ins, 75 string asmOps, list<dag> pattern=[]> : 76 InstSI<outs, ins, "", pattern>, 77 SIMCInstr<opName, SIEncodingFamily.NONE> { 78 79 let isPseudo = 1; 80 let isCodeGenOnly = 1; 81 let Size = 8; 82 let UseNamedOperandTable = 1; 83 84 string Mnemonic = opName; 85 string AsmOperands = asmOps; 86 87 let VM_CNT = 1; 88 let EXP_CNT = 1; 89 let MTBUF = 1; 90 let Uses = [EXEC]; 91 let hasSideEffects = 0; 92 let SchedRW = [WriteVMEM]; 93 94 let AsmMatchConverter = "cvtMtbuf"; 95 96 bits<1> offen = 0; 97 bits<1> idxen = 0; 98 bits<1> addr64 = 0; 99 bits<1> has_vdata = 1; 100 bits<1> has_vaddr = 1; 101 bits<1> has_glc = 1; 102 bits<1> glc_value = 0; // the value for glc if no such operand 103 bits<4> dfmt_value = 1; // the value for dfmt if no such operand 104 bits<3> nfmt_value = 0; // the value for nfmt if no such operand 105 bits<1> has_srsrc = 1; 106 bits<1> has_soffset = 1; 107 bits<1> has_offset = 1; 108 bits<1> has_slc = 1; 109 bits<1> has_tfe = 1; 110 bits<1> has_dfmt = 1; 111 bits<1> has_nfmt = 1; 112} 113 114class MTBUF_Real <MTBUF_Pseudo ps> : 115 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 116 117 let isPseudo = 0; 118 let isCodeGenOnly = 0; 119 120 // copy relevant pseudo op flags 121 let SubtargetPredicate = ps.SubtargetPredicate; 122 let AsmMatchConverter = ps.AsmMatchConverter; 123 let Constraints = ps.Constraints; 124 let DisableEncoding = ps.DisableEncoding; 125 let TSFlags = ps.TSFlags; 126 127 bits<12> offset; 128 bits<1> glc; 129 bits<4> dfmt; 130 bits<3> nfmt; 131 bits<8> vaddr; 132 bits<8> vdata; 133 bits<7> srsrc; 134 bits<1> slc; 135 bits<1> tfe; 136 bits<8> soffset; 137} 138 139class getMTBUFInsDA<list<RegisterClass> vdataList, 140 list<RegisterClass> vaddrList=[]> { 141 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 142 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 143 dag InsNoData = !if(!empty(vaddrList), 144 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 145 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, SLC:$slc, TFE:$tfe), 146 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 147 offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, SLC:$slc, TFE:$tfe) 148 ); 149 dag InsData = !if(!empty(vaddrList), 150 (ins vdataClass:$vdata, SReg_128:$srsrc, 151 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 152 SLC:$slc, TFE:$tfe), 153 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 154 SCSrc_b32:$soffset, offset:$offset, DFMT:$dfmt, NFMT:$nfmt, GLC:$glc, 155 SLC:$slc, TFE:$tfe) 156 ); 157 dag ret = !if(!empty(vdataList), InsNoData, InsData); 158} 159 160class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> { 161 dag ret = 162 !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret, 163 !if(!eq(addrKind, BUFAddrKind.OffEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 164 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMTBUFInsDA<vdataList, [VGPR_32]>.ret, 165 !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 166 !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret, 167 (ins)))))); 168} 169 170class getMTBUFAsmOps<int addrKind> { 171 string Pfx = 172 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $dfmt, $nfmt, $soffset", 173 !if(!eq(addrKind, BUFAddrKind.OffEn), 174 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset offen", 175 !if(!eq(addrKind, BUFAddrKind.IdxEn), 176 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen", 177 !if(!eq(addrKind, BUFAddrKind.BothEn), 178 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset idxen offen", 179 !if(!eq(addrKind, BUFAddrKind.Addr64), 180 "$vaddr, $srsrc, $dfmt, $nfmt, $soffset addr64", 181 ""))))); 182 string ret = Pfx # "$offset"; 183} 184 185class MTBUF_SetupAddr<int addrKind> { 186 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 187 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 188 189 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 190 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 191 192 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 193 194 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 195} 196 197class MTBUF_Load_Pseudo <string opName, 198 int addrKind, 199 RegisterClass vdataClass, 200 list<dag> pattern=[], 201 // Workaround bug bz30254 202 int addrKindCopy = addrKind> 203 : MTBUF_Pseudo<opName, 204 (outs vdataClass:$vdata), 205 getMTBUFIns<addrKindCopy>.ret, 206 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 207 pattern>, 208 MTBUF_SetupAddr<addrKindCopy> { 209 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 210 let mayLoad = 1; 211 let mayStore = 0; 212} 213 214multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 215 ValueType load_vt = i32, 216 SDPatternOperator ld = null_frag> { 217 218 def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 219 [(set load_vt:$vdata, 220 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$dfmt, 221 i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 222 MTBUFAddr64Table<0, NAME>; 223 224 def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 225 [(set load_vt:$vdata, 226 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, 227 i8:$dfmt, i8:$nfmt, i1:$glc, i1:$slc, i1:$tfe)))]>, 228 MTBUFAddr64Table<1, NAME>; 229 230 def _OFFEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 231 def _IDXEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 232 def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 233 234 let DisableWQM = 1 in { 235 def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 236 def _OFFEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 237 def _IDXEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 238 def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 239 } 240} 241 242class MTBUF_Store_Pseudo <string opName, 243 int addrKind, 244 RegisterClass vdataClass, 245 list<dag> pattern=[], 246 // Workaround bug bz30254 247 int addrKindCopy = addrKind, 248 RegisterClass vdataClassCopy = vdataClass> 249 : MTBUF_Pseudo<opName, 250 (outs), 251 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 252 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 253 pattern>, 254 MTBUF_SetupAddr<addrKindCopy> { 255 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 256 let mayLoad = 0; 257 let mayStore = 1; 258} 259 260multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 261 ValueType store_vt = i32, 262 SDPatternOperator st = null_frag> { 263 264 def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 265 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 266 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 267 i1:$slc, i1:$tfe))]>, 268 MTBUFAddr64Table<0, NAME>; 269 270 def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 271 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 272 i16:$offset, i8:$dfmt, i8:$nfmt, i1:$glc, 273 i1:$slc, i1:$tfe))]>, 274 MTBUFAddr64Table<1, NAME>; 275 276 def _OFFEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 277 def _IDXEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 278 def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 279 280 let DisableWQM = 1 in { 281 def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 282 def _OFFEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 283 def _IDXEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 284 def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 285 } 286} 287 288 289//===----------------------------------------------------------------------===// 290// MUBUF classes 291//===----------------------------------------------------------------------===// 292 293class MUBUF_Pseudo <string opName, dag outs, dag ins, 294 string asmOps, list<dag> pattern=[]> : 295 InstSI<outs, ins, "", pattern>, 296 SIMCInstr<opName, SIEncodingFamily.NONE> { 297 298 let isPseudo = 1; 299 let isCodeGenOnly = 1; 300 let Size = 8; 301 let UseNamedOperandTable = 1; 302 303 string Mnemonic = opName; 304 string AsmOperands = asmOps; 305 306 let VM_CNT = 1; 307 let EXP_CNT = 1; 308 let MUBUF = 1; 309 let Uses = [EXEC]; 310 let hasSideEffects = 0; 311 let SchedRW = [WriteVMEM]; 312 313 let AsmMatchConverter = "cvtMubuf"; 314 315 bits<1> offen = 0; 316 bits<1> idxen = 0; 317 bits<1> addr64 = 0; 318 bits<1> lds = 0; 319 bits<1> has_vdata = 1; 320 bits<1> has_vaddr = 1; 321 bits<1> has_glc = 1; 322 bits<1> glc_value = 0; // the value for glc if no such operand 323 bits<1> has_srsrc = 1; 324 bits<1> has_soffset = 1; 325 bits<1> has_offset = 1; 326 bits<1> has_slc = 1; 327 bits<1> has_tfe = 1; 328} 329 330class MUBUF_Real <bits<7> op, MUBUF_Pseudo ps> : 331 InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> { 332 333 let isPseudo = 0; 334 let isCodeGenOnly = 0; 335 336 // copy relevant pseudo op flags 337 let SubtargetPredicate = ps.SubtargetPredicate; 338 let AsmMatchConverter = ps.AsmMatchConverter; 339 let Constraints = ps.Constraints; 340 let DisableEncoding = ps.DisableEncoding; 341 let TSFlags = ps.TSFlags; 342 343 bits<12> offset; 344 bits<1> glc; 345 bits<8> vaddr; 346 bits<8> vdata; 347 bits<7> srsrc; 348 bits<1> slc; 349 bits<1> tfe; 350 bits<8> soffset; 351} 352 353 354// For cache invalidation instructions. 355class MUBUF_Invalidate <string opName, SDPatternOperator node> : 356 MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> { 357 358 let AsmMatchConverter = ""; 359 360 let hasSideEffects = 1; 361 let mayStore = 1; 362 363 // Set everything to 0. 364 let offen = 0; 365 let idxen = 0; 366 let addr64 = 0; 367 let has_vdata = 0; 368 let has_vaddr = 0; 369 let has_glc = 0; 370 let glc_value = 0; 371 let has_srsrc = 0; 372 let has_soffset = 0; 373 let has_offset = 0; 374 let has_slc = 0; 375 let has_tfe = 0; 376} 377 378class getMUBUFInsDA<list<RegisterClass> vdataList, 379 list<RegisterClass> vaddrList=[], 380 bit isLds = 0> { 381 RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList)); 382 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 383 dag InsNoData = !if(!empty(vaddrList), 384 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, 385 offset:$offset, GLC:$glc, SLC:$slc), 386 (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, 387 offset:$offset, GLC:$glc, SLC:$slc) 388 ); 389 dag InsData = !if(!empty(vaddrList), 390 (ins vdataClass:$vdata, SReg_128:$srsrc, 391 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 392 (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc, 393 SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc) 394 ); 395 dag ret = !con( 396 !if(!empty(vdataList), InsNoData, InsData), 397 !if(isLds, (ins), (ins TFE:$tfe)) 398 ); 399} 400 401class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> { 402 dag ret = 403 !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret, 404 !if(!eq(addrKind, BUFAddrKind.OffEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 405 !if(!eq(addrKind, BUFAddrKind.IdxEn), getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret, 406 !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 407 !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret, 408 (ins)))))); 409} 410 411class getMUBUFAsmOps<int addrKind> { 412 string Pfx = 413 !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset", 414 !if(!eq(addrKind, BUFAddrKind.OffEn), "$vaddr, $srsrc, $soffset offen", 415 !if(!eq(addrKind, BUFAddrKind.IdxEn), "$vaddr, $srsrc, $soffset idxen", 416 !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen", 417 !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64", 418 ""))))); 419 string ret = Pfx # "$offset"; 420} 421 422class MUBUF_SetupAddr<int addrKind> { 423 bits<1> offen = !if(!eq(addrKind, BUFAddrKind.OffEn), 1, 424 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 425 426 bits<1> idxen = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1, 427 !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0)); 428 429 bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0); 430 431 bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1); 432} 433 434class MUBUF_Load_Pseudo <string opName, 435 int addrKind, 436 RegisterClass vdataClass, 437 bit HasTiedDest = 0, 438 bit isLds = 0, 439 list<dag> pattern=[], 440 // Workaround bug bz30254 441 int addrKindCopy = addrKind> 442 : MUBUF_Pseudo<opName, 443 (outs vdataClass:$vdata), 444 !con(getMUBUFIns<addrKindCopy, [], isLds>.ret, 445 !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))), 446 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" # 447 !if(isLds, " lds", "$tfe"), 448 pattern>, 449 MUBUF_SetupAddr<addrKindCopy> { 450 let PseudoInstr = opName # !if(isLds, "_lds", "") # 451 "_" # getAddrName<addrKindCopy>.ret; 452 let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf"); 453 454 let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", ""); 455 let mayLoad = 1; 456 let mayStore = 0; 457 let maybeAtomic = 1; 458 let Uses = !if(isLds, [EXEC, M0], [EXEC]); 459 let has_tfe = !if(isLds, 0, 1); 460 let lds = isLds; 461} 462 463// FIXME: tfe can't be an operand because it requires a separate 464// opcode because it needs an N+1 register class dest register. 465multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass, 466 ValueType load_vt = i32, 467 SDPatternOperator ld = null_frag, 468 bit TiedDest = 0, 469 bit isLds = 0> { 470 471 def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 472 TiedDest, isLds, 473 !if(isLds, 474 [], 475 [(set load_vt:$vdata, 476 (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 477 MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>; 478 479 def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 480 TiedDest, isLds, 481 !if(isLds, 482 [], 483 [(set load_vt:$vdata, 484 (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe)))])>, 485 MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>; 486 487 def _OFFEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 488 def _IDXEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 489 def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 490 491 let DisableWQM = 1 in { 492 def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>; 493 def _OFFEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>; 494 def _IDXEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>; 495 def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>; 496 } 497} 498 499multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass, 500 ValueType load_vt = i32, 501 SDPatternOperator ld_nolds = null_frag, 502 SDPatternOperator ld_lds = null_frag> { 503 defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>; 504 defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>; 505} 506 507class MUBUF_Store_Pseudo <string opName, 508 int addrKind, 509 RegisterClass vdataClass, 510 list<dag> pattern=[], 511 // Workaround bug bz30254 512 int addrKindCopy = addrKind, 513 RegisterClass vdataClassCopy = vdataClass> 514 : MUBUF_Pseudo<opName, 515 (outs), 516 getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret, 517 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe", 518 pattern>, 519 MUBUF_SetupAddr<addrKindCopy> { 520 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 521 let mayLoad = 0; 522 let mayStore = 1; 523 let maybeAtomic = 1; 524} 525 526multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass, 527 ValueType store_vt = i32, 528 SDPatternOperator st = null_frag> { 529 530 def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 531 [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 532 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 533 MUBUFAddr64Table<0, NAME>; 534 535 def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 536 [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 537 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))]>, 538 MUBUFAddr64Table<1, NAME>; 539 540 def _OFFEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 541 def _IDXEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 542 def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 543 544 let DisableWQM = 1 in { 545 def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>; 546 def _OFFEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 547 def _IDXEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 548 def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 549 } 550} 551 552class MUBUF_Pseudo_Store_Lds<string opName> 553 : MUBUF_Pseudo<opName, 554 (outs), 555 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc), 556 " $srsrc, $soffset$offset lds$glc$slc"> { 557 let mayLoad = 0; 558 let mayStore = 1; 559 let maybeAtomic = 1; 560 561 let has_vdata = 0; 562 let has_vaddr = 0; 563 let has_tfe = 0; 564 let lds = 1; 565 566 let Uses = [EXEC, M0]; 567 let AsmMatchConverter = "cvtMubufLds"; 568} 569 570class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, 571 list<RegisterClass> vaddrList=[]> { 572 RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList)); 573 dag ret = !if(vdata_in, 574 !if(!empty(vaddrList), 575 (ins vdataClass:$vdata_in, 576 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 577 (ins vdataClass:$vdata_in, vaddrClass:$vaddr, 578 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 579 ), 580 !if(!empty(vaddrList), 581 (ins vdataClass:$vdata, 582 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc), 583 (ins vdataClass:$vdata, vaddrClass:$vaddr, 584 SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc) 585 )); 586} 587 588class getMUBUFAtomicIns<int addrKind, 589 RegisterClass vdataClass, 590 bit vdata_in, 591 // Workaround bug bz30254 592 RegisterClass vdataClassCopy=vdataClass> { 593 dag ret = 594 !if(!eq(addrKind, BUFAddrKind.Offset), 595 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret, 596 !if(!eq(addrKind, BUFAddrKind.OffEn), 597 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 598 !if(!eq(addrKind, BUFAddrKind.IdxEn), 599 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret, 600 !if(!eq(addrKind, BUFAddrKind.BothEn), 601 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 602 !if(!eq(addrKind, BUFAddrKind.Addr64), 603 getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret, 604 (ins)))))); 605} 606 607class MUBUF_Atomic_Pseudo<string opName, 608 int addrKind, 609 dag outs, 610 dag ins, 611 string asmOps, 612 list<dag> pattern=[], 613 // Workaround bug bz30254 614 int addrKindCopy = addrKind> 615 : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>, 616 MUBUF_SetupAddr<addrKindCopy> { 617 let mayStore = 1; 618 let mayLoad = 1; 619 let hasPostISelHook = 1; 620 let hasSideEffects = 1; 621 let DisableWQM = 1; 622 let has_glc = 0; 623 let has_tfe = 0; 624 let maybeAtomic = 1; 625} 626 627class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind, 628 RegisterClass vdataClass, 629 list<dag> pattern=[], 630 // Workaround bug bz30254 631 int addrKindCopy = addrKind, 632 RegisterClass vdataClassCopy = vdataClass> 633 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 634 (outs), 635 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret, 636 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc", 637 pattern>, 638 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> { 639 let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret; 640 let glc_value = 0; 641 let AsmMatchConverter = "cvtMubufAtomic"; 642} 643 644class MUBUF_AtomicRet_Pseudo<string opName, int addrKind, 645 RegisterClass vdataClass, 646 list<dag> pattern=[], 647 // Workaround bug bz30254 648 int addrKindCopy = addrKind, 649 RegisterClass vdataClassCopy = vdataClass> 650 : MUBUF_Atomic_Pseudo<opName, addrKindCopy, 651 (outs vdataClassCopy:$vdata), 652 getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret, 653 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc", 654 pattern>, 655 AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> { 656 let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret; 657 let glc_value = 1; 658 let Constraints = "$vdata = $vdata_in"; 659 let DisableEncoding = "$vdata_in"; 660 let AsmMatchConverter = "cvtMubufAtomicReturn"; 661} 662 663multiclass MUBUF_Pseudo_Atomics <string opName, 664 RegisterClass vdataClass, 665 ValueType vdataType, 666 SDPatternOperator atomic> { 667 668 def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>, 669 MUBUFAddr64Table <0, NAME>; 670 def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>, 671 MUBUFAddr64Table <1, NAME>; 672 def _OFFEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 673 def _IDXEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 674 def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 675 676 def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 677 [(set vdataType:$vdata, 678 (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc), 679 vdataType:$vdata_in))]>, 680 MUBUFAddr64Table <0, NAME # "_RTN">; 681 682 def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 683 [(set vdataType:$vdata, 684 (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc), 685 vdataType:$vdata_in))]>, 686 MUBUFAddr64Table <1, NAME # "_RTN">; 687 688 def _OFFEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>; 689 def _IDXEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>; 690 def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>; 691} 692 693 694//===----------------------------------------------------------------------===// 695// MUBUF Instructions 696//===----------------------------------------------------------------------===// 697 698defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds < 699 "buffer_load_format_x", VGPR_32 700>; 701defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads < 702 "buffer_load_format_xy", VReg_64 703>; 704defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads < 705 "buffer_load_format_xyz", VReg_96 706>; 707defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads < 708 "buffer_load_format_xyzw", VReg_128 709>; 710defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores < 711 "buffer_store_format_x", VGPR_32 712>; 713defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores < 714 "buffer_store_format_xy", VReg_64 715>; 716defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores < 717 "buffer_store_format_xyz", VReg_96 718>; 719defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores < 720 "buffer_store_format_xyzw", VReg_128 721>; 722 723let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 724 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads < 725 "buffer_load_format_d16_x", VGPR_32 726 >; 727 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads < 728 "buffer_load_format_d16_xy", VReg_64 729 >; 730 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads < 731 "buffer_load_format_d16_xyz", VReg_96 732 >; 733 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads < 734 "buffer_load_format_d16_xyzw", VReg_128 735 >; 736 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores < 737 "buffer_store_format_d16_x", VGPR_32 738 >; 739 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores < 740 "buffer_store_format_d16_xy", VReg_64 741 >; 742 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores < 743 "buffer_store_format_d16_xyz", VReg_96 744 >; 745 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores < 746 "buffer_store_format_d16_xyzw", VReg_128 747 >; 748} // End HasUnpackedD16VMem. 749 750let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 751 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads < 752 "buffer_load_format_d16_x", VGPR_32 753 >; 754 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads < 755 "buffer_load_format_d16_xy", VGPR_32 756 >; 757 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads < 758 "buffer_load_format_d16_xyz", VReg_64 759 >; 760 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads < 761 "buffer_load_format_d16_xyzw", VReg_64 762 >; 763 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores < 764 "buffer_store_format_d16_x", VGPR_32 765 >; 766 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores < 767 "buffer_store_format_d16_xy", VGPR_32 768 >; 769 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores < 770 "buffer_store_format_d16_xyz", VReg_64 771 >; 772 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores < 773 "buffer_store_format_d16_xyzw", VReg_64 774 >; 775} // End HasPackedD16VMem. 776 777defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds < 778 "buffer_load_ubyte", VGPR_32, i32, mubuf_az_extloadi8 779>; 780defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds < 781 "buffer_load_sbyte", VGPR_32, i32, mubuf_sextloadi8 782>; 783defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds < 784 "buffer_load_ushort", VGPR_32, i32, mubuf_az_extloadi16 785>; 786defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds < 787 "buffer_load_sshort", VGPR_32, i32, mubuf_sextloadi16 788>; 789defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds < 790 "buffer_load_dword", VGPR_32, i32, mubuf_load 791>; 792defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads < 793 "buffer_load_dwordx2", VReg_64, v2i32, mubuf_load 794>; 795defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads < 796 "buffer_load_dwordx3", VReg_96, untyped, mubuf_load 797>; 798defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads < 799 "buffer_load_dwordx4", VReg_128, v4i32, mubuf_load 800>; 801 802// This is not described in AMD documentation, 803// but 'lds' versions of these opcodes are available 804// in at least GFX8+ chips. See Bug 37653. 805let SubtargetPredicate = isVI in { 806defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads < 807 "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1 808>; 809defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads < 810 "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1 811>; 812defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads < 813 "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1 814>; 815} 816 817defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores < 818 "buffer_store_byte", VGPR_32, i32, truncstorei8_global 819>; 820defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores < 821 "buffer_store_short", VGPR_32, i32, truncstorei16_global 822>; 823defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores < 824 "buffer_store_dword", VGPR_32, i32, store_global 825>; 826defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores < 827 "buffer_store_dwordx2", VReg_64, v2i32, store_global 828>; 829defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores < 830 "buffer_store_dwordx3", VReg_96, untyped, store_global 831>; 832defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores < 833 "buffer_store_dwordx4", VReg_128, v4i32, store_global 834>; 835defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics < 836 "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global 837>; 838defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics < 839 "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag 840>; 841defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics < 842 "buffer_atomic_add", VGPR_32, i32, atomic_add_global 843>; 844defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics < 845 "buffer_atomic_sub", VGPR_32, i32, atomic_sub_global 846>; 847defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics < 848 "buffer_atomic_smin", VGPR_32, i32, atomic_min_global 849>; 850defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics < 851 "buffer_atomic_umin", VGPR_32, i32, atomic_umin_global 852>; 853defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics < 854 "buffer_atomic_smax", VGPR_32, i32, atomic_max_global 855>; 856defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics < 857 "buffer_atomic_umax", VGPR_32, i32, atomic_umax_global 858>; 859defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics < 860 "buffer_atomic_and", VGPR_32, i32, atomic_and_global 861>; 862defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics < 863 "buffer_atomic_or", VGPR_32, i32, atomic_or_global 864>; 865defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics < 866 "buffer_atomic_xor", VGPR_32, i32, atomic_xor_global 867>; 868defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics < 869 "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global 870>; 871defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics < 872 "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global 873>; 874defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics < 875 "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global 876>; 877defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics < 878 "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag 879>; 880defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics < 881 "buffer_atomic_add_x2", VReg_64, i64, atomic_add_global 882>; 883defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics < 884 "buffer_atomic_sub_x2", VReg_64, i64, atomic_sub_global 885>; 886defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics < 887 "buffer_atomic_smin_x2", VReg_64, i64, atomic_min_global 888>; 889defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics < 890 "buffer_atomic_umin_x2", VReg_64, i64, atomic_umin_global 891>; 892defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics < 893 "buffer_atomic_smax_x2", VReg_64, i64, atomic_max_global 894>; 895defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics < 896 "buffer_atomic_umax_x2", VReg_64, i64, atomic_umax_global 897>; 898defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics < 899 "buffer_atomic_and_x2", VReg_64, i64, atomic_and_global 900>; 901defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics < 902 "buffer_atomic_or_x2", VReg_64, i64, atomic_or_global 903>; 904defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics < 905 "buffer_atomic_xor_x2", VReg_64, i64, atomic_xor_global 906>; 907defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics < 908 "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global 909>; 910defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics < 911 "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global 912>; 913 914let SubtargetPredicate = isVI in { 915def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">; 916} 917 918let SubtargetPredicate = isSI in { // isn't on CI & VI 919/* 920defm BUFFER_ATOMIC_RSUB : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">; 921defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">; 922defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">; 923defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">; 924defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">; 925defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">; 926defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">; 927defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">; 928*/ 929 930def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc", 931 int_amdgcn_buffer_wbinvl1_sc>; 932} 933 934let SubtargetPredicate = HasD16LoadStore in { 935 936defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads < 937 "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1 938>; 939 940defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads < 941 "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1 942>; 943 944defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads < 945 "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1 946>; 947 948defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads < 949 "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1 950>; 951 952defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads < 953 "buffer_load_short_d16", VGPR_32, i32, null_frag, 1 954>; 955 956defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads < 957 "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1 958>; 959 960defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores < 961 "buffer_store_byte_d16_hi", VGPR_32, i32 962>; 963 964defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores < 965 "buffer_store_short_d16_hi", VGPR_32, i32 966>; 967 968defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads < 969 "buffer_load_format_d16_hi_x", VGPR_32 970>; 971defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores < 972 "buffer_store_format_d16_hi_x", VGPR_32 973>; 974 975} // End HasD16LoadStore 976 977def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1", 978 int_amdgcn_buffer_wbinvl1>; 979 980//===----------------------------------------------------------------------===// 981// MTBUF Instructions 982//===----------------------------------------------------------------------===// 983 984defm TBUFFER_LOAD_FORMAT_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_x", VGPR_32>; 985defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_xy", VReg_64>; 986defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyz", VReg_128>; 987defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_xyzw", VReg_128>; 988defm TBUFFER_STORE_FORMAT_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_x", VGPR_32>; 989defm TBUFFER_STORE_FORMAT_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy", VReg_64>; 990defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz", VReg_128>; 991defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>; 992 993let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in { 994 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 995 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VReg_64>; 996 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_96>; 997 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_128>; 998 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 999 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VReg_64>; 1000 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_96>; 1001 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>; 1002} // End HasUnpackedD16VMem. 1003 1004let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in { 1005 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_x", VGPR_32>; 1006 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xy", VGPR_32>; 1007 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyz", VReg_64>; 1008 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Pseudo_Loads <"tbuffer_load_format_d16_xyzw", VReg_64>; 1009 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x", VGPR_32>; 1010 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy", VGPR_32>; 1011 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz", VReg_64>; 1012 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>; 1013} // End HasPackedD16VMem. 1014 1015let SubtargetPredicate = isCIVI in { 1016 1017//===----------------------------------------------------------------------===// 1018// Instruction definitions for CI and newer. 1019//===----------------------------------------------------------------------===// 1020// Remaining instructions: 1021// BUFFER_LOAD_DWORDX3 1022// BUFFER_STORE_DWORDX3 1023 1024def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol", 1025 int_amdgcn_buffer_wbinvl1_vol>; 1026 1027} // End let SubtargetPredicate = isCIVI 1028 1029//===----------------------------------------------------------------------===// 1030// MUBUF Patterns 1031//===----------------------------------------------------------------------===// 1032 1033//===----------------------------------------------------------------------===// 1034// buffer_load/store_format patterns 1035//===----------------------------------------------------------------------===// 1036 1037multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1038 string opcode> { 1039 def : GCNPat< 1040 (vt (name v4i32:$rsrc, 0, 1041 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1042 imm:$glc, imm:$slc)), 1043 (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1044 (as_i1imm $glc), (as_i1imm $slc), 0) 1045 >; 1046 1047 def : GCNPat< 1048 (vt (name v4i32:$rsrc, i32:$vindex, 1049 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1050 imm:$glc, imm:$slc)), 1051 (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1052 (as_i1imm $glc), (as_i1imm $slc), 0) 1053 >; 1054 1055 def : GCNPat< 1056 (vt (name v4i32:$rsrc, 0, 1057 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1058 imm:$glc, imm:$slc)), 1059 (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1060 (as_i1imm $glc), (as_i1imm $slc), 0) 1061 >; 1062 1063 def : GCNPat< 1064 (vt (name v4i32:$rsrc, i32:$vindex, 1065 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1066 imm:$glc, imm:$slc)), 1067 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN) 1068 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1069 $rsrc, $soffset, (as_i16imm $offset), 1070 (as_i1imm $glc), (as_i1imm $slc), 0) 1071 >; 1072} 1073 1074defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">; 1075defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">; 1076defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">; 1077 1078let SubtargetPredicate = HasUnpackedD16VMem in { 1079 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">; 1080 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1081 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1082} // End HasUnpackedD16VMem. 1083 1084let SubtargetPredicate = HasPackedD16VMem in { 1085 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">; 1086 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">; 1087 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">; 1088} // End HasPackedD16VMem. 1089 1090defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">; 1091defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">; 1092defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">; 1093 1094multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1095 string opcode> { 1096 def : GCNPat< 1097 (name vt:$vdata, v4i32:$rsrc, 0, 1098 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1099 imm:$glc, imm:$slc), 1100 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset), 1101 (as_i1imm $glc), (as_i1imm $slc), 0) 1102 >; 1103 1104 def : GCNPat< 1105 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 1106 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1107 imm:$glc, imm:$slc), 1108 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1109 (as_i16imm $offset), (as_i1imm $glc), 1110 (as_i1imm $slc), 0) 1111 >; 1112 1113 def : GCNPat< 1114 (name vt:$vdata, v4i32:$rsrc, 0, 1115 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1116 imm:$glc, imm:$slc), 1117 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1118 (as_i16imm $offset), (as_i1imm $glc), 1119 (as_i1imm $slc), 0) 1120 >; 1121 1122 def : GCNPat< 1123 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 1124 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1125 imm:$glc, imm:$slc), 1126 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact) 1127 $vdata, 1128 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1129 $rsrc, $soffset, (as_i16imm $offset), 1130 (as_i1imm $glc), (as_i1imm $slc), 0) 1131 >; 1132} 1133 1134defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">; 1135defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">; 1136defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">; 1137 1138let SubtargetPredicate = HasUnpackedD16VMem in { 1139 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">; 1140 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">; 1141 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1142} // End HasUnpackedD16VMem. 1143 1144let SubtargetPredicate = HasPackedD16VMem in { 1145 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">; 1146 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">; 1147 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">; 1148} // End HasPackedD16VMem. 1149 1150defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">; 1151defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">; 1152defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">; 1153 1154//===----------------------------------------------------------------------===// 1155// buffer_atomic patterns 1156//===----------------------------------------------------------------------===// 1157 1158multiclass BufferAtomicPatterns<SDPatternOperator name, string opcode> { 1159 def : GCNPat< 1160 (name i32:$vdata_in, v4i32:$rsrc, 0, 1161 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1162 imm:$slc), 1163 (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset, 1164 (as_i16imm $offset), (as_i1imm $slc)) 1165 >; 1166 1167 def : GCNPat< 1168 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1169 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1170 imm:$slc), 1171 (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset, 1172 (as_i16imm $offset), (as_i1imm $slc)) 1173 >; 1174 1175 def : GCNPat< 1176 (name i32:$vdata_in, v4i32:$rsrc, 0, 1177 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1178 imm:$slc), 1179 (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset, 1180 (as_i16imm $offset), (as_i1imm $slc)) 1181 >; 1182 1183 def : GCNPat< 1184 (name i32:$vdata_in, v4i32:$rsrc, i32:$vindex, 1185 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1186 imm:$slc), 1187 (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN) 1188 $vdata_in, 1189 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1190 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)) 1191 >; 1192} 1193 1194defm : BufferAtomicPatterns<SIbuffer_atomic_swap, "BUFFER_ATOMIC_SWAP">; 1195defm : BufferAtomicPatterns<SIbuffer_atomic_add, "BUFFER_ATOMIC_ADD">; 1196defm : BufferAtomicPatterns<SIbuffer_atomic_sub, "BUFFER_ATOMIC_SUB">; 1197defm : BufferAtomicPatterns<SIbuffer_atomic_smin, "BUFFER_ATOMIC_SMIN">; 1198defm : BufferAtomicPatterns<SIbuffer_atomic_umin, "BUFFER_ATOMIC_UMIN">; 1199defm : BufferAtomicPatterns<SIbuffer_atomic_smax, "BUFFER_ATOMIC_SMAX">; 1200defm : BufferAtomicPatterns<SIbuffer_atomic_umax, "BUFFER_ATOMIC_UMAX">; 1201defm : BufferAtomicPatterns<SIbuffer_atomic_and, "BUFFER_ATOMIC_AND">; 1202defm : BufferAtomicPatterns<SIbuffer_atomic_or, "BUFFER_ATOMIC_OR">; 1203defm : BufferAtomicPatterns<SIbuffer_atomic_xor, "BUFFER_ATOMIC_XOR">; 1204 1205def : GCNPat< 1206 (SIbuffer_atomic_cmpswap 1207 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1208 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1209 imm:$slc), 1210 (EXTRACT_SUBREG 1211 (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN 1212 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1213 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1214 sub0) 1215>; 1216 1217def : GCNPat< 1218 (SIbuffer_atomic_cmpswap 1219 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1220 (MUBUFIntrinsicOffset i32:$soffset, i16:$offset), 1221 imm:$slc), 1222 (EXTRACT_SUBREG 1223 (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN 1224 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1225 $vindex, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1226 sub0) 1227>; 1228 1229def : GCNPat< 1230 (SIbuffer_atomic_cmpswap 1231 i32:$data, i32:$cmp, v4i32:$rsrc, 0, 1232 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1233 imm:$slc), 1234 (EXTRACT_SUBREG 1235 (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN 1236 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1237 $voffset, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1238 sub0) 1239>; 1240 1241def : GCNPat< 1242 (SIbuffer_atomic_cmpswap 1243 i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex, 1244 (MUBUFIntrinsicVOffset i32:$soffset, i16:$offset, i32:$voffset), 1245 imm:$slc), 1246 (EXTRACT_SUBREG 1247 (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN 1248 (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1), 1249 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1250 $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $slc)), 1251 sub0) 1252>; 1253 1254 1255class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt, 1256 PatFrag constant_ld> : GCNPat < 1257 (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1258 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1259 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1260 >; 1261 1262multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1263 ValueType vt, PatFrag atomic_ld> { 1264 def : GCNPat < 1265 (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1266 i16:$offset, i1:$slc))), 1267 (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1268 >; 1269 1270 def : GCNPat < 1271 (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))), 1272 (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1273 >; 1274} 1275 1276let SubtargetPredicate = isSICI in { 1277def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>; 1278def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, az_extloadi8_constant>; 1279def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>; 1280def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, az_extloadi16_constant>; 1281 1282defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, mubuf_load_atomic>; 1283defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, mubuf_load_atomic>; 1284} // End SubtargetPredicate = isSICI 1285 1286multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1287 PatFrag ld> { 1288 1289 def : GCNPat < 1290 (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1291 i16:$offset, i1:$glc, i1:$slc, i1:$tfe))), 1292 (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1293 >; 1294} 1295 1296let OtherPredicates = [Has16BitInsts] in { 1297 1298defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>; 1299defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_constant>; 1300defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, mubuf_sextloadi8>; 1301defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, mubuf_az_extloadi8>; 1302 1303defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, mubuf_load>; 1304 1305} // End OtherPredicates = [Has16BitInsts] 1306 1307multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen, 1308 MUBUF_Pseudo InstrOffset, 1309 ValueType vt, PatFrag ld> { 1310 def : GCNPat < 1311 (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1312 i32:$soffset, u16imm:$offset))), 1313 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1314 >; 1315 1316 def : GCNPat < 1317 (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1318 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0) 1319 >; 1320} 1321 1322// XXX - Is it possible to have a complex pattern in a PatFrag? 1323multiclass MUBUFScratchLoadPat_Hi16 <MUBUF_Pseudo InstrOffen, 1324 MUBUF_Pseudo InstrOffset, 1325 ValueType vt, PatFrag ld> { 1326 def : GCNPat < 1327 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1328 i32:$soffset, u16imm:$offset)))), 1329 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1330 >; 1331 1332 def : GCNPat < 1333 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1334 i32:$soffset, u16imm:$offset)))))), 1335 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1336 >; 1337 1338 1339 def : GCNPat < 1340 (build_vector vt:$lo, (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))), 1341 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1342 >; 1343 1344 def : GCNPat < 1345 (build_vector f16:$lo, (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset)))))), 1346 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $lo)) 1347 >; 1348} 1349 1350multiclass MUBUFScratchLoadPat_Lo16 <MUBUF_Pseudo InstrOffen, 1351 MUBUF_Pseudo InstrOffset, 1352 ValueType vt, PatFrag ld> { 1353 def : GCNPat < 1354 (build_vector (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1355 i32:$soffset, u16imm:$offset))), 1356 (vt (Hi16Elt vt:$hi))), 1357 (v2i16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1358 >; 1359 1360 def : GCNPat < 1361 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1362 i32:$soffset, u16imm:$offset))))), 1363 (f16 (Hi16Elt f16:$hi))), 1364 (v2f16 (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1365 >; 1366 1367 def : GCNPat < 1368 (build_vector (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), 1369 (vt (Hi16Elt vt:$hi))), 1370 (v2i16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1371 >; 1372 1373 def : GCNPat < 1374 (build_vector (f16 (bitconvert (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))))), 1375 (f16 (Hi16Elt f16:$hi))), 1376 (v2f16 (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, $hi)) 1377 >; 1378} 1379 1380defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>; 1381defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, az_extloadi8_private>; 1382defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>; 1383defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, az_extloadi8_private>; 1384defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>; 1385defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, az_extloadi16_private>; 1386defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>; 1387defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>; 1388defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>; 1389defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>; 1390 1391let OtherPredicates = [D16PreservesUnusedBits] in { 1392defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, i16, load_private>; 1393defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, i16, az_extloadi8_private>; 1394defm : MUBUFScratchLoadPat_Hi16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, i16, sextloadi8_private>; 1395 1396defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, i16, load_private>; 1397defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, i16, az_extloadi8_private>; 1398defm : MUBUFScratchLoadPat_Lo16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, i16, sextloadi8_private>; 1399} 1400 1401// BUFFER_LOAD_DWORD*, addr64=0 1402multiclass MUBUF_Load_Dword <ValueType vt, 1403 MUBUF_Pseudo offset, 1404 MUBUF_Pseudo offen, 1405 MUBUF_Pseudo idxen, 1406 MUBUF_Pseudo bothen> { 1407 1408 def : GCNPat < 1409 (vt (int_SI_buffer_load_dword v4i32:$rsrc, (i32 imm), i32:$soffset, 1410 imm:$offset, 0, 0, imm:$glc, imm:$slc, 1411 imm:$tfe)), 1412 (offset $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1413 (as_i1imm $slc), (as_i1imm $tfe)) 1414 >; 1415 1416 def : GCNPat < 1417 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1418 imm:$offset, 1, 0, imm:$glc, imm:$slc, 1419 imm:$tfe)), 1420 (offen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1421 (as_i1imm $tfe)) 1422 >; 1423 1424 def : GCNPat < 1425 (vt (int_SI_buffer_load_dword v4i32:$rsrc, i32:$vaddr, i32:$soffset, 1426 imm:$offset, 0, 1, imm:$glc, imm:$slc, 1427 imm:$tfe)), 1428 (idxen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), 1429 (as_i1imm $slc), (as_i1imm $tfe)) 1430 >; 1431 1432 def : GCNPat < 1433 (vt (int_SI_buffer_load_dword v4i32:$rsrc, v2i32:$vaddr, i32:$soffset, 1434 imm:$offset, 1, 1, imm:$glc, imm:$slc, 1435 imm:$tfe)), 1436 (bothen $vaddr, $rsrc, $soffset, (as_i16imm $offset), (as_i1imm $glc), (as_i1imm $slc), 1437 (as_i1imm $tfe)) 1438 >; 1439} 1440 1441defm : MUBUF_Load_Dword <i32, BUFFER_LOAD_DWORD_OFFSET, BUFFER_LOAD_DWORD_OFFEN, 1442 BUFFER_LOAD_DWORD_IDXEN, BUFFER_LOAD_DWORD_BOTHEN>; 1443defm : MUBUF_Load_Dword <v2i32, BUFFER_LOAD_DWORDX2_OFFSET, BUFFER_LOAD_DWORDX2_OFFEN, 1444 BUFFER_LOAD_DWORDX2_IDXEN, BUFFER_LOAD_DWORDX2_BOTHEN>; 1445defm : MUBUF_Load_Dword <v4i32, BUFFER_LOAD_DWORDX4_OFFSET, BUFFER_LOAD_DWORDX4_OFFEN, 1446 BUFFER_LOAD_DWORDX4_IDXEN, BUFFER_LOAD_DWORDX4_BOTHEN>; 1447 1448multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET, 1449 ValueType vt, PatFrag atomic_st> { 1450 // Store follows atomic op convention so address is forst 1451 def : GCNPat < 1452 (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, 1453 i16:$offset, i1:$slc), vt:$val), 1454 (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0) 1455 >; 1456 1457 def : GCNPat < 1458 (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), 1459 (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0) 1460 >; 1461} 1462let SubtargetPredicate = isSICI in { 1463defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; 1464defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; 1465} // End Predicates = isSICI 1466 1467 1468multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt, 1469 PatFrag st> { 1470 1471 def : GCNPat < 1472 (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, 1473 i16:$offset, i1:$glc, i1:$slc, i1:$tfe)), 1474 (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe) 1475 >; 1476} 1477 1478defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>; 1479defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>; 1480 1481multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen, 1482 MUBUF_Pseudo InstrOffset, 1483 ValueType vt, PatFrag st> { 1484 def : GCNPat < 1485 (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, 1486 i32:$soffset, u16imm:$offset)), 1487 (InstrOffen $value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0) 1488 >; 1489 1490 def : GCNPat < 1491 (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, 1492 u16imm:$offset)), 1493 (InstrOffset $value, $srsrc, $soffset, $offset, 0, 0, 0) 1494 >; 1495} 1496 1497defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>; 1498defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>; 1499defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>; 1500defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>; 1501defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>; 1502defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private>; 1503defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private>; 1504 1505 1506let OtherPredicates = [D16PreservesUnusedBits] in { 1507 // Hiding the extract high pattern in the PatFrag seems to not 1508 // automatically increase the complexity. 1509let AddedComplexity = 1 in { 1510defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>; 1511defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>; 1512} 1513} 1514 1515//===----------------------------------------------------------------------===// 1516// MTBUF Patterns 1517//===----------------------------------------------------------------------===// 1518 1519//===----------------------------------------------------------------------===// 1520// tbuffer_load/store_format patterns 1521//===----------------------------------------------------------------------===// 1522 1523multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt, 1524 string opcode> { 1525 def : GCNPat< 1526 (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1527 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1528 (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset), 1529 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1530 >; 1531 1532 def : GCNPat< 1533 (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1534 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1535 (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset), 1536 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1537 >; 1538 1539 def : GCNPat< 1540 (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1541 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1542 (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset), 1543 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1544 >; 1545 1546 def : GCNPat< 1547 (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset, 1548 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc)), 1549 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN) 1550 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1551 $rsrc, $soffset, (as_i16imm $offset), 1552 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1553 >; 1554} 1555 1556defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32, "TBUFFER_LOAD_FORMAT_X">; 1557defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">; 1558defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">; 1559defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32, "TBUFFER_LOAD_FORMAT_X">; 1560defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">; 1561defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">; 1562 1563let SubtargetPredicate = HasUnpackedD16VMem in { 1564 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X_gfx80">; 1565 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">; 1566 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">; 1567} // End HasUnpackedD16VMem. 1568 1569let SubtargetPredicate = HasPackedD16VMem in { 1570 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16, "TBUFFER_LOAD_FORMAT_D16_X">; 1571 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">; 1572 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">; 1573} // End HasPackedD16VMem. 1574 1575multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt, 1576 string opcode> { 1577 def : GCNPat< 1578 (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset, 1579 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1580 (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, 1581 (as_i16imm $offset), (as_i8imm $dfmt), 1582 (as_i8imm $nfmt), (as_i1imm $glc), 1583 (as_i1imm $slc), 0) 1584 >; 1585 1586 def : GCNPat< 1587 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset, 1588 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1589 (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset, 1590 (as_i16imm $offset), (as_i8imm $dfmt), 1591 (as_i8imm $nfmt), (as_i1imm $glc), 1592 (as_i1imm $slc), 0) 1593 >; 1594 1595 def : GCNPat< 1596 (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset, 1597 imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1598 (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset, 1599 (as_i16imm $offset), (as_i8imm $dfmt), 1600 (as_i8imm $nfmt), (as_i1imm $glc), 1601 (as_i1imm $slc), 0) 1602 >; 1603 1604 def : GCNPat< 1605 (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, 1606 imm:$offset, imm:$dfmt, imm:$nfmt, imm:$glc, imm:$slc), 1607 (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact) 1608 $vdata, 1609 (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1), 1610 $rsrc, $soffset, (as_i16imm $offset), 1611 (as_i8imm $dfmt), (as_i8imm $nfmt), (as_i1imm $glc), (as_i1imm $slc), 0) 1612 >; 1613} 1614 1615defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32, "TBUFFER_STORE_FORMAT_X">; 1616defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">; 1617defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4i32, "TBUFFER_STORE_FORMAT_XYZ">; 1618defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">; 1619defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32, "TBUFFER_STORE_FORMAT_X">; 1620defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">; 1621defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_x3, v4f32, "TBUFFER_STORE_FORMAT_XYZ">; 1622defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">; 1623 1624let SubtargetPredicate = HasUnpackedD16VMem in { 1625 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X_gfx80">; 1626 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">; 1627 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">; 1628} // End HasUnpackedD16VMem. 1629 1630let SubtargetPredicate = HasPackedD16VMem in { 1631 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16, "TBUFFER_STORE_FORMAT_D16_X">; 1632 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">; 1633 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">; 1634} // End HasPackedD16VMem. 1635 1636//===----------------------------------------------------------------------===// 1637// Target instructions, move to the appropriate target TD file 1638//===----------------------------------------------------------------------===// 1639 1640//===----------------------------------------------------------------------===// 1641// SI 1642//===----------------------------------------------------------------------===// 1643 1644class MUBUF_Real_si <bits<7> op, MUBUF_Pseudo ps> : 1645 MUBUF_Real<op, ps>, 1646 Enc64, 1647 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1648 let AssemblerPredicate=isSICI; 1649 let DecoderNamespace="SICI"; 1650 1651 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1652 let Inst{12} = ps.offen; 1653 let Inst{13} = ps.idxen; 1654 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1655 let Inst{15} = ps.addr64; 1656 let Inst{16} = !if(ps.lds, 1, 0); 1657 let Inst{24-18} = op; 1658 let Inst{31-26} = 0x38; //encoding 1659 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1660 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1661 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1662 let Inst{54} = !if(ps.has_slc, slc, ?); 1663 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1664 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1665} 1666 1667multiclass MUBUF_Real_AllAddr_si<bits<7> op> { 1668 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1669 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>; 1670 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1671 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1672 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1673} 1674 1675multiclass MUBUF_Real_AllAddr_Lds_si<bits<7> op> { 1676 1677 def _OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1678 MUBUFLdsTable<0, NAME # "_OFFSET_si">; 1679 def _ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>, 1680 MUBUFLdsTable<0, NAME # "_ADDR64_si">; 1681 def _OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1682 MUBUFLdsTable<0, NAME # "_OFFEN_si">; 1683 def _IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1684 MUBUFLdsTable<0, NAME # "_IDXEN_si">; 1685 def _BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1686 MUBUFLdsTable<0, NAME # "_BOTHEN_si">; 1687 1688 def _LDS_OFFSET_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1689 MUBUFLdsTable<1, NAME # "_OFFSET_si">; 1690 def _LDS_ADDR64_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>, 1691 MUBUFLdsTable<1, NAME # "_ADDR64_si">; 1692 def _LDS_OFFEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1693 MUBUFLdsTable<1, NAME # "_OFFEN_si">; 1694 def _LDS_IDXEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1695 MUBUFLdsTable<1, NAME # "_IDXEN_si">; 1696 def _LDS_BOTHEN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1697 MUBUFLdsTable<1, NAME # "_BOTHEN_si">; 1698} 1699 1700multiclass MUBUF_Real_Atomic_si<bits<7> op> : MUBUF_Real_AllAddr_si<op> { 1701 def _OFFSET_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1702 def _ADDR64_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>; 1703 def _OFFEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1704 def _IDXEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1705 def _BOTHEN_RTN_si : MUBUF_Real_si <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1706} 1707 1708defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_si <0x00>; 1709defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_si <0x01>; 1710defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x02>; 1711defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x03>; 1712defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_si <0x04>; 1713defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_si <0x05>; 1714defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_si <0x06>; 1715defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_si <0x07>; 1716defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_si <0x08>; 1717defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_si <0x09>; 1718defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_si <0x0a>; 1719defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_si <0x0b>; 1720defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_si <0x0c>; 1721defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_si <0x0d>; 1722defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_si <0x0e>; 1723defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_si <0x0f>; 1724defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_si <0x18>; 1725defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_si <0x1a>; 1726defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_si <0x1c>; 1727defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_si <0x1d>; 1728defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_si <0x1e>; 1729defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_si <0x1f>; 1730 1731defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_si <0x30>; 1732defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_si <0x31>; 1733defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_si <0x32>; 1734defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_si <0x33>; 1735//defm BUFFER_ATOMIC_RSUB : MUBUF_Real_Atomic_si <0x34>; // isn't on CI & VI 1736defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_si <0x35>; 1737defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_si <0x36>; 1738defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_si <0x37>; 1739defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_si <0x38>; 1740defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_si <0x39>; 1741defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_si <0x3a>; 1742defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_si <0x3b>; 1743defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_si <0x3c>; 1744defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_si <0x3d>; 1745 1746//defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Real_Atomic_si <0x3e>; // isn't on VI 1747//defm BUFFER_ATOMIC_FMIN : MUBUF_Real_Atomic_si <0x3f>; // isn't on VI 1748//defm BUFFER_ATOMIC_FMAX : MUBUF_Real_Atomic_si <0x40>; // isn't on VI 1749defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_si <0x50>; 1750defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_si <0x51>; 1751defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_si <0x52>; 1752defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_si <0x53>; 1753//defm BUFFER_ATOMIC_RSUB_X2 : MUBUF_Real_Atomic_si <0x54>; // isn't on CI & VI 1754defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_si <0x55>; 1755defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_si <0x56>; 1756defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_si <0x57>; 1757defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_si <0x58>; 1758defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_si <0x59>; 1759defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_si <0x5a>; 1760defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_si <0x5b>; 1761defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_si <0x5c>; 1762defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_si <0x5d>; 1763// FIXME: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on CI. 1764//defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomic_si <0x5e">; // isn't on VI 1765//defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Real_Atomic_si <0x5f>; // isn't on VI 1766//defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Real_Atomic_si <0x60>; // isn't on VI 1767 1768def BUFFER_WBINVL1_SC_si : MUBUF_Real_si <0x70, BUFFER_WBINVL1_SC>; 1769def BUFFER_WBINVL1_si : MUBUF_Real_si <0x71, BUFFER_WBINVL1>; 1770 1771class MTBUF_Real_si <bits<3> op, MTBUF_Pseudo ps> : 1772 MTBUF_Real<ps>, 1773 Enc64, 1774 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> { 1775 let AssemblerPredicate=isSICI; 1776 let DecoderNamespace="SICI"; 1777 1778 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1779 let Inst{12} = ps.offen; 1780 let Inst{13} = ps.idxen; 1781 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1782 let Inst{15} = ps.addr64; 1783 let Inst{18-16} = op; 1784 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 1785 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 1786 let Inst{31-26} = 0x3a; //encoding 1787 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1788 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1789 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1790 let Inst{54} = !if(ps.has_slc, slc, ?); 1791 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1792 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1793} 1794 1795multiclass MTBUF_Real_AllAddr_si<bits<3> op> { 1796 def _OFFSET_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 1797 def _ADDR64_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>; 1798 def _OFFEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 1799 def _IDXEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 1800 def _BOTHEN_si : MTBUF_Real_si <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 1801} 1802 1803defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_si <0>; 1804defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_si <1>; 1805defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_si <2>; 1806defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_si <3>; 1807defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_si <4>; 1808defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_si <5>; 1809defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_si <6>; 1810defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_si <7>; 1811 1812//===----------------------------------------------------------------------===// 1813// CI 1814//===----------------------------------------------------------------------===// 1815 1816class MUBUF_Real_ci <bits<7> op, MUBUF_Pseudo ps> : 1817 MUBUF_Real_si<op, ps> { 1818 let AssemblerPredicate=isCIOnly; 1819 let DecoderNamespace="CI"; 1820} 1821 1822def BUFFER_WBINVL1_VOL_ci : MUBUF_Real_ci <0x70, BUFFER_WBINVL1_VOL>; 1823 1824 1825//===----------------------------------------------------------------------===// 1826// VI 1827//===----------------------------------------------------------------------===// 1828 1829class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> : 1830 MUBUF_Real<op, ps>, 1831 Enc64, 1832 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 1833 let AssemblerPredicate=isVI; 1834 let DecoderNamespace="VI"; 1835 1836 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1837 let Inst{12} = ps.offen; 1838 let Inst{13} = ps.idxen; 1839 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1840 let Inst{16} = !if(ps.lds, 1, 0); 1841 let Inst{17} = !if(ps.has_slc, slc, ?); 1842 let Inst{24-18} = op; 1843 let Inst{31-26} = 0x38; //encoding 1844 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1845 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1846 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1847 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1848 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1849} 1850 1851multiclass MUBUF_Real_AllAddr_vi<bits<7> op> { 1852 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1853 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1854 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1855 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1856} 1857 1858multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> { 1859 1860 def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>, 1861 MUBUFLdsTable<0, NAME # "_OFFSET_vi">; 1862 def _OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>, 1863 MUBUFLdsTable<0, NAME # "_OFFEN_vi">; 1864 def _IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>, 1865 MUBUFLdsTable<0, NAME # "_IDXEN_vi">; 1866 def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>, 1867 MUBUFLdsTable<0, NAME # "_BOTHEN_vi">; 1868 1869 def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>, 1870 MUBUFLdsTable<1, NAME # "_OFFSET_vi">; 1871 def _LDS_OFFEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>, 1872 MUBUFLdsTable<1, NAME # "_OFFEN_vi">; 1873 def _LDS_IDXEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>, 1874 MUBUFLdsTable<1, NAME # "_IDXEN_vi">; 1875 def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>, 1876 MUBUFLdsTable<1, NAME # "_BOTHEN_vi">; 1877} 1878 1879class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> : 1880 MUBUF_Real<op, ps>, 1881 Enc64, 1882 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 1883 let AssemblerPredicate=HasUnpackedD16VMem; 1884 let DecoderNamespace="GFX80_UNPACKED"; 1885 1886 let Inst{11-0} = !if(ps.has_offset, offset, ?); 1887 let Inst{12} = ps.offen; 1888 let Inst{13} = ps.idxen; 1889 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 1890 let Inst{16} = !if(ps.lds, 1, 0); 1891 let Inst{17} = !if(ps.has_slc, slc, ?); 1892 let Inst{24-18} = op; 1893 let Inst{31-26} = 0x38; //encoding 1894 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 1895 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 1896 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 1897 let Inst{55} = !if(ps.has_tfe, tfe, ?); 1898 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 1899} 1900 1901multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> { 1902 def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>; 1903 def _OFFEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>; 1904 def _IDXEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>; 1905 def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>; 1906} 1907 1908multiclass MUBUF_Real_Atomic_vi<bits<7> op> : 1909 MUBUF_Real_AllAddr_vi<op> { 1910 def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>; 1911 def _OFFEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>; 1912 def _IDXEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>; 1913 def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>; 1914} 1915 1916defm BUFFER_LOAD_FORMAT_X : MUBUF_Real_AllAddr_Lds_vi <0x00>; 1917defm BUFFER_LOAD_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x01>; 1918defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x02>; 1919defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x03>; 1920defm BUFFER_STORE_FORMAT_X : MUBUF_Real_AllAddr_vi <0x04>; 1921defm BUFFER_STORE_FORMAT_XY : MUBUF_Real_AllAddr_vi <0x05>; 1922defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Real_AllAddr_vi <0x06>; 1923defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_vi <0x07>; 1924let SubtargetPredicate = HasUnpackedD16VMem in { 1925 defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x08>; 1926 defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x09>; 1927 defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0a>; 1928 defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0b>; 1929 defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0c>; 1930 defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0d>; 1931 defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0e>; 1932 defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Real_AllAddr_gfx80 <0x0f>; 1933} // End HasUnpackedD16VMem. 1934let SubtargetPredicate = HasPackedD16VMem in { 1935 defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x08>; 1936 defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x09>; 1937 defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0a>; 1938 defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0b>; 1939 defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Real_AllAddr_vi <0x0c>; 1940 defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Real_AllAddr_vi <0x0d>; 1941 defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Real_AllAddr_vi <0x0e>; 1942 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_vi <0x0f>; 1943} // End HasPackedD16VMem. 1944defm BUFFER_LOAD_UBYTE : MUBUF_Real_AllAddr_Lds_vi <0x10>; 1945defm BUFFER_LOAD_SBYTE : MUBUF_Real_AllAddr_Lds_vi <0x11>; 1946defm BUFFER_LOAD_USHORT : MUBUF_Real_AllAddr_Lds_vi <0x12>; 1947defm BUFFER_LOAD_SSHORT : MUBUF_Real_AllAddr_Lds_vi <0x13>; 1948defm BUFFER_LOAD_DWORD : MUBUF_Real_AllAddr_Lds_vi <0x14>; 1949defm BUFFER_LOAD_DWORDX2 : MUBUF_Real_AllAddr_Lds_vi <0x15>; 1950defm BUFFER_LOAD_DWORDX3 : MUBUF_Real_AllAddr_Lds_vi <0x16>; 1951defm BUFFER_LOAD_DWORDX4 : MUBUF_Real_AllAddr_Lds_vi <0x17>; 1952defm BUFFER_STORE_BYTE : MUBUF_Real_AllAddr_vi <0x18>; 1953defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x19>; 1954defm BUFFER_STORE_SHORT : MUBUF_Real_AllAddr_vi <0x1a>; 1955defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x1b>; 1956defm BUFFER_STORE_DWORD : MUBUF_Real_AllAddr_vi <0x1c>; 1957defm BUFFER_STORE_DWORDX2 : MUBUF_Real_AllAddr_vi <0x1d>; 1958defm BUFFER_STORE_DWORDX3 : MUBUF_Real_AllAddr_vi <0x1e>; 1959defm BUFFER_STORE_DWORDX4 : MUBUF_Real_AllAddr_vi <0x1f>; 1960 1961defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Real_AllAddr_vi <0x20>; 1962defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x21>; 1963defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Real_AllAddr_vi <0x22>; 1964defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Real_AllAddr_vi <0x23>; 1965defm BUFFER_LOAD_SHORT_D16 : MUBUF_Real_AllAddr_vi <0x24>; 1966defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Real_AllAddr_vi <0x25>; 1967 1968defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x26>; 1969defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>; 1970 1971defm BUFFER_ATOMIC_SWAP : MUBUF_Real_Atomic_vi <0x40>; 1972defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Real_Atomic_vi <0x41>; 1973defm BUFFER_ATOMIC_ADD : MUBUF_Real_Atomic_vi <0x42>; 1974defm BUFFER_ATOMIC_SUB : MUBUF_Real_Atomic_vi <0x43>; 1975defm BUFFER_ATOMIC_SMIN : MUBUF_Real_Atomic_vi <0x44>; 1976defm BUFFER_ATOMIC_UMIN : MUBUF_Real_Atomic_vi <0x45>; 1977defm BUFFER_ATOMIC_SMAX : MUBUF_Real_Atomic_vi <0x46>; 1978defm BUFFER_ATOMIC_UMAX : MUBUF_Real_Atomic_vi <0x47>; 1979defm BUFFER_ATOMIC_AND : MUBUF_Real_Atomic_vi <0x48>; 1980defm BUFFER_ATOMIC_OR : MUBUF_Real_Atomic_vi <0x49>; 1981defm BUFFER_ATOMIC_XOR : MUBUF_Real_Atomic_vi <0x4a>; 1982defm BUFFER_ATOMIC_INC : MUBUF_Real_Atomic_vi <0x4b>; 1983defm BUFFER_ATOMIC_DEC : MUBUF_Real_Atomic_vi <0x4c>; 1984 1985defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Real_Atomic_vi <0x60>; 1986defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Real_Atomic_vi <0x61>; 1987defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Real_Atomic_vi <0x62>; 1988defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Real_Atomic_vi <0x63>; 1989defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Real_Atomic_vi <0x64>; 1990defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Real_Atomic_vi <0x65>; 1991defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Real_Atomic_vi <0x66>; 1992defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Real_Atomic_vi <0x67>; 1993defm BUFFER_ATOMIC_AND_X2 : MUBUF_Real_Atomic_vi <0x68>; 1994defm BUFFER_ATOMIC_OR_X2 : MUBUF_Real_Atomic_vi <0x69>; 1995defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Real_Atomic_vi <0x6a>; 1996defm BUFFER_ATOMIC_INC_X2 : MUBUF_Real_Atomic_vi <0x6b>; 1997defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Real_Atomic_vi <0x6c>; 1998 1999def BUFFER_STORE_LDS_DWORD_vi : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>; 2000 2001def BUFFER_WBINVL1_vi : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>; 2002def BUFFER_WBINVL1_VOL_vi : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>; 2003 2004class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> : 2005 MTBUF_Real<ps>, 2006 Enc64, 2007 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> { 2008 let AssemblerPredicate=isVI; 2009 let DecoderNamespace="VI"; 2010 2011 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2012 let Inst{12} = ps.offen; 2013 let Inst{13} = ps.idxen; 2014 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2015 let Inst{18-15} = op; 2016 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 2017 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 2018 let Inst{31-26} = 0x3a; //encoding 2019 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2020 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2021 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2022 let Inst{54} = !if(ps.has_slc, slc, ?); 2023 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2024 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2025} 2026 2027multiclass MTBUF_Real_AllAddr_vi<bits<4> op> { 2028 def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2029 def _OFFEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2030 def _IDXEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2031 def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2032} 2033 2034class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> : 2035 MTBUF_Real<ps>, 2036 Enc64, 2037 SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> { 2038 let AssemblerPredicate=HasUnpackedD16VMem; 2039 let DecoderNamespace="GFX80_UNPACKED"; 2040 2041 let Inst{11-0} = !if(ps.has_offset, offset, ?); 2042 let Inst{12} = ps.offen; 2043 let Inst{13} = ps.idxen; 2044 let Inst{14} = !if(ps.has_glc, glc, ps.glc_value); 2045 let Inst{18-15} = op; 2046 let Inst{22-19} = !if(ps.has_dfmt, dfmt, ps.dfmt_value); 2047 let Inst{25-23} = !if(ps.has_nfmt, nfmt, ps.nfmt_value); 2048 let Inst{31-26} = 0x3a; //encoding 2049 let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?); 2050 let Inst{47-40} = !if(ps.has_vdata, vdata, ?); 2051 let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?); 2052 let Inst{54} = !if(ps.has_slc, slc, ?); 2053 let Inst{55} = !if(ps.has_tfe, tfe, ?); 2054 let Inst{63-56} = !if(ps.has_soffset, soffset, ?); 2055} 2056 2057multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> { 2058 def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>; 2059 def _OFFEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>; 2060 def _IDXEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>; 2061 def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>; 2062} 2063 2064defm TBUFFER_LOAD_FORMAT_X : MTBUF_Real_AllAddr_vi <0x00>; 2065defm TBUFFER_LOAD_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x01>; 2066defm TBUFFER_LOAD_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x02>; 2067defm TBUFFER_LOAD_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x03>; 2068defm TBUFFER_STORE_FORMAT_X : MTBUF_Real_AllAddr_vi <0x04>; 2069defm TBUFFER_STORE_FORMAT_XY : MTBUF_Real_AllAddr_vi <0x05>; 2070defm TBUFFER_STORE_FORMAT_XYZ : MTBUF_Real_AllAddr_vi <0x06>; 2071defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>; 2072let SubtargetPredicate = HasUnpackedD16VMem in { 2073 defm TBUFFER_LOAD_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x08>; 2074 defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x09>; 2075 defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0a>; 2076 defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0b>; 2077 defm TBUFFER_STORE_FORMAT_D16_X_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0c>; 2078 defm TBUFFER_STORE_FORMAT_D16_XY_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0d>; 2079 defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0e>; 2080 defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>; 2081} // End HasUnpackedD16VMem. 2082let SubtargetPredicate = HasPackedD16VMem in { 2083 defm TBUFFER_LOAD_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x08>; 2084 defm TBUFFER_LOAD_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x09>; 2085 defm TBUFFER_LOAD_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0a>; 2086 defm TBUFFER_LOAD_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0b>; 2087 defm TBUFFER_STORE_FORMAT_D16_X : MTBUF_Real_AllAddr_vi <0x0c>; 2088 defm TBUFFER_STORE_FORMAT_D16_XY : MTBUF_Real_AllAddr_vi <0x0d>; 2089 defm TBUFFER_STORE_FORMAT_D16_XYZ : MTBUF_Real_AllAddr_vi <0x0e>; 2090 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>; 2091} // End HasUnpackedD16VMem. 2092