1 //===- X86EvexToVex.cpp ---------------------------------------------------===//
2 // Compress EVEX instructions to VEX encoding when possible to reduce code size
3 //
4 // The LLVM Compiler Infrastructure
5 //
6 // This file is distributed under the University of Illinois Open Source
7 // License. See LICENSE.TXT for details.
8 //
9 //===----------------------------------------------------------------------===//
10 //
11 /// \file
12 /// This file defines the pass that goes over all AVX-512 instructions which
13 /// are encoded using the EVEX prefix and if possible replaces them by their
14 /// corresponding VEX encoding which is usually shorter by 2 bytes.
15 /// EVEX instructions may be encoded via the VEX prefix when the AVX-512
16 /// instruction has a corresponding AVX/AVX2 opcode and when it does not
17 /// use the xmm or the mask registers or xmm/ymm registers with indexes
18 /// higher than 15.
19 /// The pass applies code reduction on the generated code for AVX-512 instrs.
20 //
21 //===----------------------------------------------------------------------===//
22
23 #include "InstPrinter/X86InstComments.h"
24 #include "MCTargetDesc/X86BaseInfo.h"
25 #include "X86.h"
26 #include "X86InstrInfo.h"
27 #include "X86Subtarget.h"
28 #include "llvm/ADT/StringRef.h"
29 #include "llvm/CodeGen/MachineFunction.h"
30 #include "llvm/CodeGen/MachineFunctionPass.h"
31 #include "llvm/CodeGen/MachineInstr.h"
32 #include "llvm/CodeGen/MachineOperand.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/Pass.h"
35 #include <cassert>
36 #include <cstdint>
37
38 using namespace llvm;
39
40 // Including the generated EVEX2VEX tables.
41 struct X86EvexToVexCompressTableEntry {
42 uint16_t EvexOpcode;
43 uint16_t VexOpcode;
44
operator <X86EvexToVexCompressTableEntry45 bool operator<(const X86EvexToVexCompressTableEntry &RHS) const {
46 return EvexOpcode < RHS.EvexOpcode;
47 }
48
operator <(const X86EvexToVexCompressTableEntry & TE,unsigned Opc)49 friend bool operator<(const X86EvexToVexCompressTableEntry &TE,
50 unsigned Opc) {
51 return TE.EvexOpcode < Opc;
52 }
53 };
54 #include "X86GenEVEX2VEXTables.inc"
55
56 #define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible"
57 #define EVEX2VEX_NAME "x86-evex-to-vex-compress"
58
59 #define DEBUG_TYPE EVEX2VEX_NAME
60
61 namespace {
62
63 class EvexToVexInstPass : public MachineFunctionPass {
64
65 /// For EVEX instructions that can be encoded using VEX encoding, replace
66 /// them by the VEX encoding in order to reduce size.
67 bool CompressEvexToVexImpl(MachineInstr &MI) const;
68
69 public:
70 static char ID;
71
EvexToVexInstPass()72 EvexToVexInstPass() : MachineFunctionPass(ID) {
73 initializeEvexToVexInstPassPass(*PassRegistry::getPassRegistry());
74 }
75
getPassName() const76 StringRef getPassName() const override { return EVEX2VEX_DESC; }
77
78 /// Loop over all of the basic blocks, replacing EVEX instructions
79 /// by equivalent VEX instructions when possible for reducing code size.
80 bool runOnMachineFunction(MachineFunction &MF) override;
81
82 // This pass runs after regalloc and doesn't support VReg operands.
getRequiredProperties() const83 MachineFunctionProperties getRequiredProperties() const override {
84 return MachineFunctionProperties().set(
85 MachineFunctionProperties::Property::NoVRegs);
86 }
87
88 private:
89 /// Machine instruction info used throughout the class.
90 const X86InstrInfo *TII;
91 };
92
93 } // end anonymous namespace
94
95 char EvexToVexInstPass::ID = 0;
96
runOnMachineFunction(MachineFunction & MF)97 bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) {
98 TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
99
100 const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
101 if (!ST.hasAVX512())
102 return false;
103
104 bool Changed = false;
105
106 /// Go over all basic blocks in function and replace
107 /// EVEX encoded instrs by VEX encoding when possible.
108 for (MachineBasicBlock &MBB : MF) {
109
110 // Traverse the basic block.
111 for (MachineInstr &MI : MBB)
112 Changed |= CompressEvexToVexImpl(MI);
113 }
114
115 return Changed;
116 }
117
usesExtendedRegister(const MachineInstr & MI)118 static bool usesExtendedRegister(const MachineInstr &MI) {
119 auto isHiRegIdx = [](unsigned Reg) {
120 // Check for XMM register with indexes between 16 - 31.
121 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
122 return true;
123
124 // Check for YMM register with indexes between 16 - 31.
125 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
126 return true;
127
128 return false;
129 };
130
131 // Check that operands are not ZMM regs or
132 // XMM/YMM regs with hi indexes between 16 - 31.
133 for (const MachineOperand &MO : MI.explicit_operands()) {
134 if (!MO.isReg())
135 continue;
136
137 unsigned Reg = MO.getReg();
138
139 assert(!(Reg >= X86::ZMM0 && Reg <= X86::ZMM31) &&
140 "ZMM instructions should not be in the EVEX->VEX tables");
141
142 if (isHiRegIdx(Reg))
143 return true;
144 }
145
146 return false;
147 }
148
149 // Do any custom cleanup needed to finalize the conversion.
performCustomAdjustments(MachineInstr & MI,unsigned NewOpc)150 static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc) {
151 (void)NewOpc;
152 unsigned Opc = MI.getOpcode();
153 switch (Opc) {
154 case X86::VALIGNDZ128rri:
155 case X86::VALIGNDZ128rmi:
156 case X86::VALIGNQZ128rri:
157 case X86::VALIGNQZ128rmi: {
158 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
159 "Unexpected new opcode!");
160 unsigned Scale = (Opc == X86::VALIGNQZ128rri ||
161 Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
162 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
163 Imm.setImm(Imm.getImm() * Scale);
164 break;
165 }
166 case X86::VSHUFF32X4Z256rmi:
167 case X86::VSHUFF32X4Z256rri:
168 case X86::VSHUFF64X2Z256rmi:
169 case X86::VSHUFF64X2Z256rri:
170 case X86::VSHUFI32X4Z256rmi:
171 case X86::VSHUFI32X4Z256rri:
172 case X86::VSHUFI64X2Z256rmi:
173 case X86::VSHUFI64X2Z256rri: {
174 assert((NewOpc == X86::VPERM2F128rr || NewOpc == X86::VPERM2I128rr ||
175 NewOpc == X86::VPERM2F128rm || NewOpc == X86::VPERM2I128rm) &&
176 "Unexpected new opcode!");
177 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
178 int64_t ImmVal = Imm.getImm();
179 // Set bit 5, move bit 1 to bit 4, copy bit 0.
180 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
181 break;
182 }
183 case X86::VRNDSCALEPDZ128rri:
184 case X86::VRNDSCALEPDZ128rmi:
185 case X86::VRNDSCALEPSZ128rri:
186 case X86::VRNDSCALEPSZ128rmi:
187 case X86::VRNDSCALEPDZ256rri:
188 case X86::VRNDSCALEPDZ256rmi:
189 case X86::VRNDSCALEPSZ256rri:
190 case X86::VRNDSCALEPSZ256rmi:
191 case X86::VRNDSCALESDZr:
192 case X86::VRNDSCALESDZm:
193 case X86::VRNDSCALESSZr:
194 case X86::VRNDSCALESSZm:
195 case X86::VRNDSCALESDZr_Int:
196 case X86::VRNDSCALESDZm_Int:
197 case X86::VRNDSCALESSZr_Int:
198 case X86::VRNDSCALESSZm_Int:
199 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
200 int64_t ImmVal = Imm.getImm();
201 // Ensure that only bits 3:0 of the immediate are used.
202 if ((ImmVal & 0xf) != ImmVal)
203 return false;
204 break;
205 }
206
207 return true;
208 }
209
210
211 // For EVEX instructions that can be encoded using VEX encoding
212 // replace them by the VEX encoding in order to reduce size.
CompressEvexToVexImpl(MachineInstr & MI) const213 bool EvexToVexInstPass::CompressEvexToVexImpl(MachineInstr &MI) const {
214 // VEX format.
215 // # of bytes: 0,2,3 1 1 0,1 0,1,2,4 0,1
216 // [Prefixes] [VEX] OPCODE ModR/M [SIB] [DISP] [IMM]
217 //
218 // EVEX format.
219 // # of bytes: 4 1 1 1 4 / 1 1
220 // [Prefixes] EVEX Opcode ModR/M [SIB] [Disp32] / [Disp8*N] [Immediate]
221
222 const MCInstrDesc &Desc = MI.getDesc();
223
224 // Check for EVEX instructions only.
225 if ((Desc.TSFlags & X86II::EncodingMask) != X86II::EVEX)
226 return false;
227
228 // Check for EVEX instructions with mask or broadcast as in these cases
229 // the EVEX prefix is needed in order to carry this information
230 // thus preventing the transformation to VEX encoding.
231 if (Desc.TSFlags & (X86II::EVEX_K | X86II::EVEX_B))
232 return false;
233
234 // Check for EVEX instructions with L2 set. These instructions are 512-bits
235 // and can't be converted to VEX.
236 if (Desc.TSFlags & X86II::EVEX_L2)
237 return false;
238
239 #ifndef NDEBUG
240 // Make sure the tables are sorted.
241 static std::atomic<bool> TableChecked(false);
242 if (!TableChecked.load(std::memory_order_relaxed)) {
243 assert(std::is_sorted(std::begin(X86EvexToVex128CompressTable),
244 std::end(X86EvexToVex128CompressTable)) &&
245 "X86EvexToVex128CompressTable is not sorted!");
246 assert(std::is_sorted(std::begin(X86EvexToVex256CompressTable),
247 std::end(X86EvexToVex256CompressTable)) &&
248 "X86EvexToVex256CompressTable is not sorted!");
249 TableChecked.store(true, std::memory_order_relaxed);
250 }
251 #endif
252
253 // Use the VEX.L bit to select the 128 or 256-bit table.
254 ArrayRef<X86EvexToVexCompressTableEntry> Table =
255 (Desc.TSFlags & X86II::VEX_L) ? makeArrayRef(X86EvexToVex256CompressTable)
256 : makeArrayRef(X86EvexToVex128CompressTable);
257
258 auto I = std::lower_bound(Table.begin(), Table.end(), MI.getOpcode());
259 if (I == Table.end() || I->EvexOpcode != MI.getOpcode())
260 return false;
261
262 unsigned NewOpc = I->VexOpcode;
263
264 if (usesExtendedRegister(MI))
265 return false;
266
267 if (!performCustomAdjustments(MI, NewOpc))
268 return false;
269
270 MI.setDesc(TII->get(NewOpc));
271 MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
272 return true;
273 }
274
INITIALIZE_PASS(EvexToVexInstPass,EVEX2VEX_NAME,EVEX2VEX_DESC,false,false)275 INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false)
276
277 FunctionPass *llvm::createX86EvexToVexInsts() {
278 return new EvexToVexInstPass();
279 }
280