• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // On Fermi, image handles are not supported. To work around this, we traverse
11 // the machine code and replace image handles with concrete symbols. For this
12 // to work reliably, inlining of all function call must be performed.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "NVPTX.h"
17 #include "NVPTXMachineFunctionInfo.h"
18 #include "NVPTXSubtarget.h"
19 #include "NVPTXTargetMachine.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/Support/raw_ostream.h"
25 
26 using namespace llvm;
27 
28 namespace {
29 class NVPTXReplaceImageHandles : public MachineFunctionPass {
30 private:
31   static char ID;
32   DenseSet<MachineInstr *> InstrsToRemove;
33 
34 public:
35   NVPTXReplaceImageHandles();
36 
37   bool runOnMachineFunction(MachineFunction &MF) override;
38 
getPassName() const39   const char *getPassName() const override {
40     return "NVPTX Replace Image Handles";
41   }
42 private:
43   bool processInstr(MachineInstr &MI);
44   void replaceImageHandle(MachineOperand &Op, MachineFunction &MF);
45   bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF,
46                           unsigned &Idx);
47 };
48 }
49 
50 char NVPTXReplaceImageHandles::ID = 0;
51 
NVPTXReplaceImageHandles()52 NVPTXReplaceImageHandles::NVPTXReplaceImageHandles()
53   : MachineFunctionPass(ID) {}
54 
runOnMachineFunction(MachineFunction & MF)55 bool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) {
56   bool Changed = false;
57   InstrsToRemove.clear();
58 
59   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
60        ++BI) {
61     for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end();
62          I != E; ++I) {
63       MachineInstr &MI = *I;
64       Changed |= processInstr(MI);
65     }
66   }
67 
68   // Now clean up any handle-access instructions
69   // This is needed in debug mode when code cleanup passes are not executed,
70   // but we need the handle access to be eliminated because they are not
71   // valid instructions when image handles are disabled.
72   for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(),
73        E = InstrsToRemove.end(); I != E; ++I) {
74     (*I)->eraseFromParent();
75   }
76   return Changed;
77 }
78 
processInstr(MachineInstr & MI)79 bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
80   MachineFunction &MF = *MI.getParent()->getParent();
81   const MCInstrDesc &MCID = MI.getDesc();
82 
83   if (MCID.TSFlags & NVPTXII::IsTexFlag) {
84     // This is a texture fetch, so operand 4 is a texref and operand 5 is
85     // a samplerref
86     MachineOperand &TexHandle = MI.getOperand(4);
87     replaceImageHandle(TexHandle, MF);
88 
89     if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
90       MachineOperand &SampHandle = MI.getOperand(5);
91       replaceImageHandle(SampHandle, MF);
92     }
93 
94     return true;
95   } else if (MCID.TSFlags & NVPTXII::IsSuldMask) {
96     unsigned VecSize =
97       1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1);
98 
99     // For a surface load of vector size N, the Nth operand will be the surfref
100     MachineOperand &SurfHandle = MI.getOperand(VecSize);
101 
102     replaceImageHandle(SurfHandle, MF);
103 
104     return true;
105   } else if (MCID.TSFlags & NVPTXII::IsSustFlag) {
106     // This is a surface store, so operand 0 is a surfref
107     MachineOperand &SurfHandle = MI.getOperand(0);
108 
109     replaceImageHandle(SurfHandle, MF);
110 
111     return true;
112   } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
113     // This is a query, so operand 1 is a surfref/texref
114     MachineOperand &Handle = MI.getOperand(1);
115 
116     replaceImageHandle(Handle, MF);
117 
118     return true;
119   }
120 
121   return false;
122 }
123 
124 void NVPTXReplaceImageHandles::
replaceImageHandle(MachineOperand & Op,MachineFunction & MF)125 replaceImageHandle(MachineOperand &Op, MachineFunction &MF) {
126   unsigned Idx;
127   if (findIndexForHandle(Op, MF, Idx)) {
128     Op.ChangeToImmediate(Idx);
129   }
130 }
131 
132 bool NVPTXReplaceImageHandles::
findIndexForHandle(MachineOperand & Op,MachineFunction & MF,unsigned & Idx)133 findIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) {
134   const MachineRegisterInfo &MRI = MF.getRegInfo();
135   NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
136 
137   assert(Op.isReg() && "Handle is not in a reg?");
138 
139   // Which instruction defines the handle?
140   MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg());
141 
142   switch (TexHandleDef.getOpcode()) {
143   case NVPTX::LD_i64_avar: {
144     // The handle is a parameter value being loaded, replace with the
145     // parameter symbol
146     const NVPTXTargetMachine &TM =
147         static_cast<const NVPTXTargetMachine &>(MF.getTarget());
148     if (TM.getDrvInterface() == NVPTX::CUDA) {
149       // For CUDA, we preserve the param loads coming from function arguments
150       return false;
151     }
152 
153     assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!");
154     StringRef Sym = TexHandleDef.getOperand(6).getSymbolName();
155     std::string ParamBaseName = MF.getName();
156     ParamBaseName += "_param_";
157     assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference");
158     unsigned Param = atoi(Sym.data()+ParamBaseName.size());
159     std::string NewSym;
160     raw_string_ostream NewSymStr(NewSym);
161     NewSymStr << MF.getFunction()->getName() << "_param_" << Param;
162 
163     InstrsToRemove.insert(&TexHandleDef);
164     Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str());
165     return true;
166   }
167   case NVPTX::texsurf_handles: {
168     // The handle is a global variable, replace with the global variable name
169     assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
170     const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal();
171     assert(GV->hasName() && "Global sampler must be named!");
172     InstrsToRemove.insert(&TexHandleDef);
173     Idx = MFI->getImageHandleSymbolIndex(GV->getName().data());
174     return true;
175   }
176   case NVPTX::nvvm_move_i64:
177   case TargetOpcode::COPY: {
178     bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx);
179     if (Res) {
180       InstrsToRemove.insert(&TexHandleDef);
181     }
182     return Res;
183   }
184   default:
185     llvm_unreachable("Unknown instruction operating on handle");
186   }
187 }
188 
createNVPTXReplaceImageHandlesPass()189 MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() {
190   return new NVPTXReplaceImageHandles();
191 }
192