1 //===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // On Fermi, image handles are not supported. To work around this, we traverse
11 // the machine code and replace image handles with concrete symbols. For this
12 // to work reliably, inlining of all function call must be performed.
13 //
14 //===----------------------------------------------------------------------===//
15
16 #include "NVPTX.h"
17 #include "NVPTXMachineFunctionInfo.h"
18 #include "NVPTXSubtarget.h"
19 #include "NVPTXTargetMachine.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/CodeGen/MachineFunction.h"
22 #include "llvm/CodeGen/MachineFunctionPass.h"
23 #include "llvm/CodeGen/MachineRegisterInfo.h"
24 #include "llvm/Support/raw_ostream.h"
25
26 using namespace llvm;
27
28 namespace {
29 class NVPTXReplaceImageHandles : public MachineFunctionPass {
30 private:
31 static char ID;
32 DenseSet<MachineInstr *> InstrsToRemove;
33
34 public:
35 NVPTXReplaceImageHandles();
36
37 bool runOnMachineFunction(MachineFunction &MF) override;
38
getPassName() const39 const char *getPassName() const override {
40 return "NVPTX Replace Image Handles";
41 }
42 private:
43 bool processInstr(MachineInstr &MI);
44 void replaceImageHandle(MachineOperand &Op, MachineFunction &MF);
45 bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF,
46 unsigned &Idx);
47 };
48 }
49
50 char NVPTXReplaceImageHandles::ID = 0;
51
NVPTXReplaceImageHandles()52 NVPTXReplaceImageHandles::NVPTXReplaceImageHandles()
53 : MachineFunctionPass(ID) {}
54
runOnMachineFunction(MachineFunction & MF)55 bool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) {
56 bool Changed = false;
57 InstrsToRemove.clear();
58
59 for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
60 ++BI) {
61 for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end();
62 I != E; ++I) {
63 MachineInstr &MI = *I;
64 Changed |= processInstr(MI);
65 }
66 }
67
68 // Now clean up any handle-access instructions
69 // This is needed in debug mode when code cleanup passes are not executed,
70 // but we need the handle access to be eliminated because they are not
71 // valid instructions when image handles are disabled.
72 for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(),
73 E = InstrsToRemove.end(); I != E; ++I) {
74 (*I)->eraseFromParent();
75 }
76 return Changed;
77 }
78
processInstr(MachineInstr & MI)79 bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
80 MachineFunction &MF = *MI.getParent()->getParent();
81 const MCInstrDesc &MCID = MI.getDesc();
82
83 if (MCID.TSFlags & NVPTXII::IsTexFlag) {
84 // This is a texture fetch, so operand 4 is a texref and operand 5 is
85 // a samplerref
86 MachineOperand &TexHandle = MI.getOperand(4);
87 replaceImageHandle(TexHandle, MF);
88
89 if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
90 MachineOperand &SampHandle = MI.getOperand(5);
91 replaceImageHandle(SampHandle, MF);
92 }
93
94 return true;
95 } else if (MCID.TSFlags & NVPTXII::IsSuldMask) {
96 unsigned VecSize =
97 1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1);
98
99 // For a surface load of vector size N, the Nth operand will be the surfref
100 MachineOperand &SurfHandle = MI.getOperand(VecSize);
101
102 replaceImageHandle(SurfHandle, MF);
103
104 return true;
105 } else if (MCID.TSFlags & NVPTXII::IsSustFlag) {
106 // This is a surface store, so operand 0 is a surfref
107 MachineOperand &SurfHandle = MI.getOperand(0);
108
109 replaceImageHandle(SurfHandle, MF);
110
111 return true;
112 } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
113 // This is a query, so operand 1 is a surfref/texref
114 MachineOperand &Handle = MI.getOperand(1);
115
116 replaceImageHandle(Handle, MF);
117
118 return true;
119 }
120
121 return false;
122 }
123
124 void NVPTXReplaceImageHandles::
replaceImageHandle(MachineOperand & Op,MachineFunction & MF)125 replaceImageHandle(MachineOperand &Op, MachineFunction &MF) {
126 unsigned Idx;
127 if (findIndexForHandle(Op, MF, Idx)) {
128 Op.ChangeToImmediate(Idx);
129 }
130 }
131
132 bool NVPTXReplaceImageHandles::
findIndexForHandle(MachineOperand & Op,MachineFunction & MF,unsigned & Idx)133 findIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) {
134 const MachineRegisterInfo &MRI = MF.getRegInfo();
135 NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
136
137 assert(Op.isReg() && "Handle is not in a reg?");
138
139 // Which instruction defines the handle?
140 MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg());
141
142 switch (TexHandleDef.getOpcode()) {
143 case NVPTX::LD_i64_avar: {
144 // The handle is a parameter value being loaded, replace with the
145 // parameter symbol
146 const NVPTXTargetMachine &TM =
147 static_cast<const NVPTXTargetMachine &>(MF.getTarget());
148 if (TM.getDrvInterface() == NVPTX::CUDA) {
149 // For CUDA, we preserve the param loads coming from function arguments
150 return false;
151 }
152
153 assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!");
154 StringRef Sym = TexHandleDef.getOperand(6).getSymbolName();
155 std::string ParamBaseName = MF.getName();
156 ParamBaseName += "_param_";
157 assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference");
158 unsigned Param = atoi(Sym.data()+ParamBaseName.size());
159 std::string NewSym;
160 raw_string_ostream NewSymStr(NewSym);
161 NewSymStr << MF.getFunction()->getName() << "_param_" << Param;
162
163 InstrsToRemove.insert(&TexHandleDef);
164 Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str());
165 return true;
166 }
167 case NVPTX::texsurf_handles: {
168 // The handle is a global variable, replace with the global variable name
169 assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
170 const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal();
171 assert(GV->hasName() && "Global sampler must be named!");
172 InstrsToRemove.insert(&TexHandleDef);
173 Idx = MFI->getImageHandleSymbolIndex(GV->getName().data());
174 return true;
175 }
176 case NVPTX::nvvm_move_i64:
177 case TargetOpcode::COPY: {
178 bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx);
179 if (Res) {
180 InstrsToRemove.insert(&TexHandleDef);
181 }
182 return Res;
183 }
184 default:
185 llvm_unreachable("Unknown instruction operating on handle");
186 }
187 }
188
createNVPTXReplaceImageHandlesPass()189 MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() {
190 return new NVPTXReplaceImageHandles();
191 }
192