1 /**************************************************************************** 2 * Copyright (C) 2014-2018 Intel Corporation. All Rights Reserved. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * @file builder_gfx_mem.cpp 24 * 25 * @brief Definition of the gfx mem builder 26 * 27 * Notes: 28 * 29 ******************************************************************************/ 30 #include "jit_pch.hpp" 31 #include "builder.h" 32 #include "common/rdtsc_buckets.h" 33 #include "builder_gfx_mem.h" 34 35 namespace SwrJit 36 { 37 using namespace llvm; 38 BuilderGfxMem(JitManager * pJitMgr)39 BuilderGfxMem::BuilderGfxMem(JitManager* pJitMgr) : Builder(pJitMgr) 40 { 41 mpTranslationFuncTy = nullptr; 42 mpfnTranslateGfxAddressForRead = nullptr; 43 mpfnTranslateGfxAddressForWrite = nullptr; 44 mpfnTrackMemAccess = nullptr; 45 mpParamSimDC = nullptr; 46 mpWorkerData = nullptr; 47 48 } 49 NotifyPrivateContextSet()50 void BuilderGfxMem::NotifyPrivateContextSet() 51 { 52 } 53 AssertGFXMemoryParams(Value * ptr,MEM_CLIENT usage)54 void BuilderGfxMem::AssertGFXMemoryParams(Value* ptr, MEM_CLIENT usage) 55 { 56 SWR_ASSERT(!(ptr->getType() == mInt64Ty && usage == MEM_CLIENT::MEM_CLIENT_INTERNAL), 57 "Internal memory should not be gfxptr_t."); 58 } 59 60 ////////////////////////////////////////////////////////////////////////// 61 /// @brief Generate a masked gather operation in LLVM IR. If not 62 /// supported on the underlying platform, emulate it with loads 63 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid 64 /// @param pBase - Int8* base VB address pointer value 65 /// @param vIndices - SIMD wide value of VB byte offsets 66 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values 67 /// @param scale - value to scale indices by GATHERPS(Value * vSrc,Value * pBase,Value * vIndices,Value * vMask,uint8_t scale,MEM_CLIENT usage)68 Value* BuilderGfxMem::GATHERPS(Value* vSrc, 69 Value* pBase, 70 Value* vIndices, 71 Value* vMask, 72 uint8_t scale, 73 MEM_CLIENT usage) 74 { 75 // address may be coming in as 64bit int now so get the pointer 76 if (pBase->getType() == mInt64Ty) 77 { 78 pBase = INT_TO_PTR(pBase, PointerType::get(mInt8Ty, 0)); 79 } 80 81 Value* vGather = Builder::GATHERPS(vSrc, pBase, vIndices, vMask, scale); 82 return vGather; 83 } 84 85 ////////////////////////////////////////////////////////////////////////// 86 /// @brief Generate a masked gather operation in LLVM IR. If not 87 /// supported on the underlying platform, emulate it with loads 88 /// @param vSrc - SIMD wide value that will be loaded if mask is invalid 89 /// @param pBase - Int8* base VB address pointer value 90 /// @param vIndices - SIMD wide value of VB byte offsets 91 /// @param vMask - SIMD wide mask that controls whether to access memory or the src values 92 /// @param scale - value to scale indices by GATHERDD(Value * vSrc,Value * pBase,Value * vIndices,Value * vMask,uint8_t scale,MEM_CLIENT usage)93 Value* BuilderGfxMem::GATHERDD(Value* vSrc, 94 Value* pBase, 95 Value* vIndices, 96 Value* vMask, 97 uint8_t scale, 98 MEM_CLIENT usage) 99 { 100 101 // address may be coming in as 64bit int now so get the pointer 102 if (pBase->getType() == mInt64Ty) 103 { 104 pBase = INT_TO_PTR(pBase, PointerType::get(mInt8Ty, 0)); 105 } 106 107 Value* vGather = Builder::GATHERDD(vSrc, pBase, vIndices, vMask, scale); 108 return vGather; 109 } 110 SCATTERPS(Value * pDst,Value * vSrc,Value * vOffsets,Value * vMask,MEM_CLIENT usage)111 void BuilderGfxMem::SCATTERPS( 112 Value* pDst, Value* vSrc, Value* vOffsets, Value* vMask, MEM_CLIENT usage) 113 { 114 115 // address may be coming in as 64bit int now so get the pointer 116 if (pDst->getType() == mInt64Ty) 117 { 118 pDst = INT_TO_PTR(pDst, PointerType::get(mInt8Ty, 0)); 119 } 120 121 Builder::SCATTERPS(pDst, BITCAST(vSrc, mSimdFP32Ty), vOffsets, vMask, usage); 122 } 123 OFFSET_TO_NEXT_COMPONENT(Value * base,Constant * offset)124 Value* BuilderGfxMem::OFFSET_TO_NEXT_COMPONENT(Value* base, Constant* offset) 125 { 126 return ADD(base, offset); 127 } 128 GEP(Value * Ptr,Value * Idx,Type * Ty,bool isReadOnly,const Twine & Name)129 Value* BuilderGfxMem::GEP(Value* Ptr, Value* Idx, Type* Ty, bool isReadOnly, const Twine& Name) 130 { 131 bool xlate = (Ptr->getType() == mInt64Ty); 132 if (xlate) 133 { 134 Ptr = INT_TO_PTR(Ptr, Ty); 135 Ptr = Builder::GEP(Ptr, Idx, nullptr, isReadOnly, Name); 136 Ptr = PTR_TO_INT(Ptr, mInt64Ty); 137 if (isReadOnly) 138 { 139 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 140 } 141 else 142 { 143 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForWrite); 144 } 145 } 146 else 147 { 148 Ptr = Builder::GEP(Ptr, Idx, nullptr, isReadOnly, Name); 149 } 150 return Ptr; 151 } 152 GEP(Type * Ty,Value * Ptr,Value * Idx,const Twine & Name)153 Value* BuilderGfxMem::GEP(Type* Ty, Value* Ptr, Value* Idx, const Twine& Name) 154 { 155 bool xlate = (Ptr->getType() == mInt64Ty); 156 if (xlate) 157 { 158 Ptr = INT_TO_PTR(Ptr, Ty); 159 Ptr = Builder::GEP(Ty, Ptr, Idx, Name); 160 Ptr = PTR_TO_INT(Ptr, mInt64Ty); 161 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 162 } 163 else 164 { 165 Ptr = Builder::GEP(Ty, Ptr, Idx, Name); 166 } 167 return Ptr; 168 } 169 GEP(Value * Ptr,const std::initializer_list<Value * > & indexList,Type * Ty)170 Value* BuilderGfxMem::GEP(Value* Ptr, const std::initializer_list<Value*>& indexList, Type* Ty) 171 { 172 bool xlate = (Ptr->getType() == mInt64Ty); 173 if (xlate) 174 { 175 Ptr = INT_TO_PTR(Ptr, Ty); 176 Ptr = Builder::GEP(Ptr, indexList); 177 Ptr = PTR_TO_INT(Ptr, mInt64Ty); 178 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 179 } 180 else 181 { 182 Ptr = Builder::GEP(Ptr, indexList); 183 } 184 return Ptr; 185 } 186 187 Value* GEP(Value * Ptr,const std::initializer_list<uint32_t> & indexList,Type * Ty)188 BuilderGfxMem::GEP(Value* Ptr, const std::initializer_list<uint32_t>& indexList, Type* Ty) 189 { 190 bool xlate = (Ptr->getType() == mInt64Ty); 191 if (xlate) 192 { 193 Ptr = INT_TO_PTR(Ptr, Ty); 194 Ptr = Builder::GEP(Ptr, indexList); 195 Ptr = PTR_TO_INT(Ptr, mInt64Ty); 196 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 197 } 198 else 199 { 200 Ptr = Builder::GEP(Ptr, indexList); 201 } 202 return Ptr; 203 } 204 TranslationHelper(Value * Ptr,Type * Ty,Value * pfnTranslateGfxAddress)205 Value* BuilderGfxMem::TranslationHelper(Value* Ptr, Type* Ty, Value* pfnTranslateGfxAddress) 206 { 207 SWR_ASSERT(!(Ptr->getType() == mInt64Ty && Ty == nullptr), 208 "Access of GFX pointers must have non-null type specified."); 209 210 // address may be coming in as 64bit int now so get the pointer 211 if (Ptr->getType() == mInt64Ty) 212 { 213 Ptr = INT_TO_PTR(Ptr, Ty); 214 } 215 216 return Ptr; 217 } 218 TrackerHelper(Value * Ptr,Type * Ty,MEM_CLIENT usage,bool isRead)219 void BuilderGfxMem::TrackerHelper(Value* Ptr, Type* Ty, MEM_CLIENT usage, bool isRead) 220 { 221 #if defined(KNOB_ENABLE_AR) 222 if (!KNOB_AR_ENABLE_MEMORY_EVENTS) 223 { 224 return; 225 } 226 227 Value* tmpPtr; 228 // convert actual pointers to int64. 229 uint32_t size = 0; 230 231 if (Ptr->getType() == mInt64Ty) 232 { 233 DataLayout dataLayout(JM()->mpCurrentModule); 234 size = (uint32_t)dataLayout.getTypeAllocSize(Ty); 235 236 tmpPtr = Ptr; 237 } 238 else 239 { 240 DataLayout dataLayout(JM()->mpCurrentModule); 241 size = (uint32_t)dataLayout.getTypeAllocSize(Ptr->getType()); 242 243 tmpPtr = PTR_TO_INT(Ptr, mInt64Ty); 244 } 245 246 // There are some shader compile setups where there's no translation functions set up. 247 // This would be a situation where the accesses are to internal rasterizer memory and won't 248 // be logged. 249 // TODO: we may wish to revisit this for URB reads/writes, though. 250 if (mpfnTrackMemAccess) 251 { 252 SWR_ASSERT(mpWorkerData != nullptr); 253 CALL(mpfnTrackMemAccess, 254 {mpParamSimDC, 255 mpWorkerData, 256 tmpPtr, 257 C((uint32_t)size), 258 C((uint8_t)isRead), 259 C((uint32_t)usage)}); 260 } 261 #endif 262 263 return; 264 } 265 LOAD(Value * Ptr,const char * Name,Type * Ty,MEM_CLIENT usage)266 LoadInst* BuilderGfxMem::LOAD(Value* Ptr, const char* Name, Type* Ty, MEM_CLIENT usage) 267 { 268 AssertGFXMemoryParams(Ptr, usage); 269 TrackerHelper(Ptr, Ty, usage, true); 270 271 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 272 return Builder::LOAD(Ptr, Name); 273 } 274 LOAD(Value * Ptr,const Twine & Name,Type * Ty,MEM_CLIENT usage)275 LoadInst* BuilderGfxMem::LOAD(Value* Ptr, const Twine& Name, Type* Ty, MEM_CLIENT usage) 276 { 277 AssertGFXMemoryParams(Ptr, usage); 278 TrackerHelper(Ptr, Ty, usage, true); 279 280 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 281 return Builder::LOAD(Ptr, Name); 282 } 283 LOAD(Value * Ptr,bool isVolatile,const Twine & Name,Type * Ty,MEM_CLIENT usage)284 LoadInst* BuilderGfxMem::LOAD( 285 Value* Ptr, bool isVolatile, const Twine& Name, Type* Ty, MEM_CLIENT usage) 286 { 287 AssertGFXMemoryParams(Ptr, usage); 288 TrackerHelper(Ptr, Ty, usage, true); 289 290 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 291 return Builder::LOAD(Ptr, isVolatile, Name); 292 } 293 LOAD(Value * BasePtr,const std::initializer_list<uint32_t> & offset,const llvm::Twine & name,Type * Ty,MEM_CLIENT usage)294 LoadInst* BuilderGfxMem::LOAD(Value* BasePtr, 295 const std::initializer_list<uint32_t>& offset, 296 const llvm::Twine& name, 297 Type* Ty, 298 MEM_CLIENT usage) 299 { 300 AssertGFXMemoryParams(BasePtr, usage); 301 302 bool bNeedTranslation = false; 303 if (BasePtr->getType() == mInt64Ty) 304 { 305 SWR_ASSERT(Ty); 306 BasePtr = INT_TO_PTR(BasePtr, Ty, name); 307 bNeedTranslation = true; 308 } 309 std::vector<Value*> valIndices; 310 for (auto i : offset) 311 { 312 valIndices.push_back(C(i)); 313 } 314 BasePtr = Builder::GEPA(BasePtr, valIndices, name); 315 if (bNeedTranslation) 316 { 317 BasePtr = PTR_TO_INT(BasePtr, mInt64Ty, name); 318 } 319 320 return LOAD(BasePtr, name, Ty, usage); 321 } 322 MASKED_LOAD(Value * Ptr,unsigned Align,Value * Mask,Value * PassThru,const Twine & Name,Type * Ty,MEM_CLIENT usage)323 CallInst* BuilderGfxMem::MASKED_LOAD(Value* Ptr, 324 unsigned Align, 325 Value* Mask, 326 Value* PassThru, 327 const Twine& Name, 328 Type* Ty, 329 MEM_CLIENT usage) 330 { 331 AssertGFXMemoryParams(Ptr, usage); 332 TrackerHelper(Ptr, Ty, usage, true); 333 334 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 335 return Builder::MASKED_LOAD(Ptr, Align, Mask, PassThru, Name, Ty, usage); 336 } 337 338 StoreInst* STORE(Value * Val,Value * Ptr,bool isVolatile,Type * Ty,MEM_CLIENT usage)339 BuilderGfxMem::STORE(Value* Val, Value* Ptr, bool isVolatile, Type* Ty, MEM_CLIENT usage) 340 { 341 AssertGFXMemoryParams(Ptr, usage); 342 TrackerHelper(Ptr, Ty, usage, false); 343 344 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 345 return Builder::STORE(Val, Ptr, isVolatile, Ty, usage); 346 } 347 STORE(Value * Val,Value * BasePtr,const std::initializer_list<uint32_t> & offset,Type * Ty,MEM_CLIENT usage)348 StoreInst* BuilderGfxMem::STORE(Value* Val, 349 Value* BasePtr, 350 const std::initializer_list<uint32_t>& offset, 351 Type* Ty, 352 MEM_CLIENT usage) 353 { 354 AssertGFXMemoryParams(BasePtr, usage); 355 TrackerHelper(BasePtr, Ty, usage, false); 356 357 BasePtr = TranslationHelper(BasePtr, Ty, mpfnTranslateGfxAddressForRead); 358 return Builder::STORE(Val, BasePtr, offset, Ty, usage); 359 } 360 MASKED_STORE(Value * Val,Value * Ptr,unsigned Align,Value * Mask,Type * Ty,MEM_CLIENT usage)361 CallInst* BuilderGfxMem::MASKED_STORE( 362 Value* Val, Value* Ptr, unsigned Align, Value* Mask, Type* Ty, MEM_CLIENT usage) 363 { 364 AssertGFXMemoryParams(Ptr, usage); 365 366 TrackerHelper(Ptr, Ty, usage, false); 367 368 Ptr = TranslationHelper(Ptr, Ty, mpfnTranslateGfxAddressForRead); 369 return Builder::MASKED_STORE(Val, Ptr, Align, Mask, Ty, usage); 370 } 371 TranslateGfxAddressForRead(Value * xpGfxAddress,Type * PtrTy,const Twine & Name,MEM_CLIENT)372 Value* BuilderGfxMem::TranslateGfxAddressForRead(Value* xpGfxAddress, 373 Type* PtrTy, 374 const Twine& Name, 375 MEM_CLIENT /* usage */) 376 { 377 if (PtrTy == nullptr) 378 { 379 PtrTy = mInt8PtrTy; 380 } 381 return INT_TO_PTR(xpGfxAddress, PtrTy, Name); 382 } 383 TranslateGfxAddressForWrite(Value * xpGfxAddress,Type * PtrTy,const Twine & Name,MEM_CLIENT)384 Value* BuilderGfxMem::TranslateGfxAddressForWrite(Value* xpGfxAddress, 385 Type* PtrTy, 386 const Twine& Name, 387 MEM_CLIENT /* usage */) 388 { 389 if (PtrTy == nullptr) 390 { 391 PtrTy = mInt8PtrTy; 392 } 393 return INT_TO_PTR(xpGfxAddress, PtrTy, Name); 394 } 395 396 } // namespace SwrJit 397