1 //===- X86EvexToVex.cpp ---------------------------------------------------===//
2 // Compress EVEX instructions to VEX encoding when possible to reduce code size
3 //
4 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // See https://llvm.org/LICENSE.txt for license information.
6 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// This file defines the pass that goes over all AVX-512 instructions which
12 /// are encoded using the EVEX prefix and if possible replaces them by their
13 /// corresponding VEX encoding which is usually shorter by 2 bytes.
14 /// EVEX instructions may be encoded via the VEX prefix when the AVX-512
15 /// instruction has a corresponding AVX/AVX2 opcode, when vector length
16 /// accessed by instruction is less than 512 bits and when it does not use
17 // the xmm or the mask registers or xmm/ymm registers with indexes higher than 15.
18 /// The pass applies code reduction on the generated code for AVX-512 instrs.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "MCTargetDesc/X86BaseInfo.h"
23 #include "MCTargetDesc/X86InstComments.h"
24 #include "X86.h"
25 #include "X86InstrInfo.h"
26 #include "X86Subtarget.h"
27 #include "llvm/ADT/StringRef.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineFunctionPass.h"
30 #include "llvm/CodeGen/MachineInstr.h"
31 #include "llvm/CodeGen/MachineOperand.h"
32 #include "llvm/MC/MCInstrDesc.h"
33 #include "llvm/Pass.h"
34 #include <cassert>
35 #include <cstdint>
36
37 using namespace llvm;
38
39 // Including the generated EVEX2VEX tables.
40 struct X86EvexToVexCompressTableEntry {
41 uint16_t EvexOpcode;
42 uint16_t VexOpcode;
43
operator <X86EvexToVexCompressTableEntry44 bool operator<(const X86EvexToVexCompressTableEntry &RHS) const {
45 return EvexOpcode < RHS.EvexOpcode;
46 }
47
operator <(const X86EvexToVexCompressTableEntry & TE,unsigned Opc)48 friend bool operator<(const X86EvexToVexCompressTableEntry &TE,
49 unsigned Opc) {
50 return TE.EvexOpcode < Opc;
51 }
52 };
53 #include "X86GenEVEX2VEXTables.inc"
54
55 #define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible"
56 #define EVEX2VEX_NAME "x86-evex-to-vex-compress"
57
58 #define DEBUG_TYPE EVEX2VEX_NAME
59
60 namespace {
61
62 class EvexToVexInstPass : public MachineFunctionPass {
63
64 /// For EVEX instructions that can be encoded using VEX encoding, replace
65 /// them by the VEX encoding in order to reduce size.
66 bool CompressEvexToVexImpl(MachineInstr &MI) const;
67
68 public:
69 static char ID;
70
EvexToVexInstPass()71 EvexToVexInstPass() : MachineFunctionPass(ID) { }
72
getPassName() const73 StringRef getPassName() const override { return EVEX2VEX_DESC; }
74
75 /// Loop over all of the basic blocks, replacing EVEX instructions
76 /// by equivalent VEX instructions when possible for reducing code size.
77 bool runOnMachineFunction(MachineFunction &MF) override;
78
79 // This pass runs after regalloc and doesn't support VReg operands.
getRequiredProperties() const80 MachineFunctionProperties getRequiredProperties() const override {
81 return MachineFunctionProperties().set(
82 MachineFunctionProperties::Property::NoVRegs);
83 }
84
85 private:
86 /// Machine instruction info used throughout the class.
87 const X86InstrInfo *TII = nullptr;
88
89 const X86Subtarget *ST = nullptr;
90 };
91
92 } // end anonymous namespace
93
94 char EvexToVexInstPass::ID = 0;
95
runOnMachineFunction(MachineFunction & MF)96 bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) {
97 TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
98
99 ST = &MF.getSubtarget<X86Subtarget>();
100 if (!ST->hasAVX512())
101 return false;
102
103 bool Changed = false;
104
105 /// Go over all basic blocks in function and replace
106 /// EVEX encoded instrs by VEX encoding when possible.
107 for (MachineBasicBlock &MBB : MF) {
108
109 // Traverse the basic block.
110 for (MachineInstr &MI : MBB)
111 Changed |= CompressEvexToVexImpl(MI);
112 }
113
114 return Changed;
115 }
116
usesExtendedRegister(const MachineInstr & MI)117 static bool usesExtendedRegister(const MachineInstr &MI) {
118 auto isHiRegIdx = [](unsigned Reg) {
119 // Check for XMM register with indexes between 16 - 31.
120 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
121 return true;
122
123 // Check for YMM register with indexes between 16 - 31.
124 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
125 return true;
126
127 return false;
128 };
129
130 // Check that operands are not ZMM regs or
131 // XMM/YMM regs with hi indexes between 16 - 31.
132 for (const MachineOperand &MO : MI.explicit_operands()) {
133 if (!MO.isReg())
134 continue;
135
136 Register Reg = MO.getReg();
137
138 assert(!(Reg >= X86::ZMM0 && Reg <= X86::ZMM31) &&
139 "ZMM instructions should not be in the EVEX->VEX tables");
140
141 if (isHiRegIdx(Reg))
142 return true;
143 }
144
145 return false;
146 }
147
148 // Do any custom cleanup needed to finalize the conversion.
performCustomAdjustments(MachineInstr & MI,unsigned NewOpc,const X86Subtarget * ST)149 static bool performCustomAdjustments(MachineInstr &MI, unsigned NewOpc,
150 const X86Subtarget *ST) {
151 (void)NewOpc;
152 unsigned Opc = MI.getOpcode();
153 switch (Opc) {
154 case X86::VPDPBUSDSZ256m:
155 case X86::VPDPBUSDSZ256r:
156 case X86::VPDPBUSDSZ128m:
157 case X86::VPDPBUSDSZ128r:
158 case X86::VPDPBUSDZ256m:
159 case X86::VPDPBUSDZ256r:
160 case X86::VPDPBUSDZ128m:
161 case X86::VPDPBUSDZ128r:
162 case X86::VPDPWSSDSZ256m:
163 case X86::VPDPWSSDSZ256r:
164 case X86::VPDPWSSDSZ128m:
165 case X86::VPDPWSSDSZ128r:
166 case X86::VPDPWSSDZ256m:
167 case X86::VPDPWSSDZ256r:
168 case X86::VPDPWSSDZ128m:
169 case X86::VPDPWSSDZ128r:
170 // These can only VEX convert if AVXVNNI is enabled.
171 return ST->hasAVXVNNI();
172 case X86::VALIGNDZ128rri:
173 case X86::VALIGNDZ128rmi:
174 case X86::VALIGNQZ128rri:
175 case X86::VALIGNQZ128rmi: {
176 assert((NewOpc == X86::VPALIGNRrri || NewOpc == X86::VPALIGNRrmi) &&
177 "Unexpected new opcode!");
178 unsigned Scale = (Opc == X86::VALIGNQZ128rri ||
179 Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
180 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
181 Imm.setImm(Imm.getImm() * Scale);
182 break;
183 }
184 case X86::VSHUFF32X4Z256rmi:
185 case X86::VSHUFF32X4Z256rri:
186 case X86::VSHUFF64X2Z256rmi:
187 case X86::VSHUFF64X2Z256rri:
188 case X86::VSHUFI32X4Z256rmi:
189 case X86::VSHUFI32X4Z256rri:
190 case X86::VSHUFI64X2Z256rmi:
191 case X86::VSHUFI64X2Z256rri: {
192 assert((NewOpc == X86::VPERM2F128rr || NewOpc == X86::VPERM2I128rr ||
193 NewOpc == X86::VPERM2F128rm || NewOpc == X86::VPERM2I128rm) &&
194 "Unexpected new opcode!");
195 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
196 int64_t ImmVal = Imm.getImm();
197 // Set bit 5, move bit 1 to bit 4, copy bit 0.
198 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
199 break;
200 }
201 case X86::VRNDSCALEPDZ128rri:
202 case X86::VRNDSCALEPDZ128rmi:
203 case X86::VRNDSCALEPSZ128rri:
204 case X86::VRNDSCALEPSZ128rmi:
205 case X86::VRNDSCALEPDZ256rri:
206 case X86::VRNDSCALEPDZ256rmi:
207 case X86::VRNDSCALEPSZ256rri:
208 case X86::VRNDSCALEPSZ256rmi:
209 case X86::VRNDSCALESDZr:
210 case X86::VRNDSCALESDZm:
211 case X86::VRNDSCALESSZr:
212 case X86::VRNDSCALESSZm:
213 case X86::VRNDSCALESDZr_Int:
214 case X86::VRNDSCALESDZm_Int:
215 case X86::VRNDSCALESSZr_Int:
216 case X86::VRNDSCALESSZm_Int:
217 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
218 int64_t ImmVal = Imm.getImm();
219 // Ensure that only bits 3:0 of the immediate are used.
220 if ((ImmVal & 0xf) != ImmVal)
221 return false;
222 break;
223 }
224
225 return true;
226 }
227
228
229 // For EVEX instructions that can be encoded using VEX encoding
230 // replace them by the VEX encoding in order to reduce size.
CompressEvexToVexImpl(MachineInstr & MI) const231 bool EvexToVexInstPass::CompressEvexToVexImpl(MachineInstr &MI) const {
232 // VEX format.
233 // # of bytes: 0,2,3 1 1 0,1 0,1,2,4 0,1
234 // [Prefixes] [VEX] OPCODE ModR/M [SIB] [DISP] [IMM]
235 //
236 // EVEX format.
237 // # of bytes: 4 1 1 1 4 / 1 1
238 // [Prefixes] EVEX Opcode ModR/M [SIB] [Disp32] / [Disp8*N] [Immediate]
239
240 const MCInstrDesc &Desc = MI.getDesc();
241
242 // Check for EVEX instructions only.
243 if ((Desc.TSFlags & X86II::EncodingMask) != X86II::EVEX)
244 return false;
245
246 // Check for EVEX instructions with mask or broadcast as in these cases
247 // the EVEX prefix is needed in order to carry this information
248 // thus preventing the transformation to VEX encoding.
249 if (Desc.TSFlags & (X86II::EVEX_K | X86II::EVEX_B))
250 return false;
251
252 // Check for EVEX instructions with L2 set. These instructions are 512-bits
253 // and can't be converted to VEX.
254 if (Desc.TSFlags & X86II::EVEX_L2)
255 return false;
256
257 #ifndef NDEBUG
258 // Make sure the tables are sorted.
259 static std::atomic<bool> TableChecked(false);
260 if (!TableChecked.load(std::memory_order_relaxed)) {
261 assert(llvm::is_sorted(X86EvexToVex128CompressTable) &&
262 "X86EvexToVex128CompressTable is not sorted!");
263 assert(llvm::is_sorted(X86EvexToVex256CompressTable) &&
264 "X86EvexToVex256CompressTable is not sorted!");
265 TableChecked.store(true, std::memory_order_relaxed);
266 }
267 #endif
268
269 // Use the VEX.L bit to select the 128 or 256-bit table.
270 ArrayRef<X86EvexToVexCompressTableEntry> Table =
271 (Desc.TSFlags & X86II::VEX_L) ? makeArrayRef(X86EvexToVex256CompressTable)
272 : makeArrayRef(X86EvexToVex128CompressTable);
273
274 const auto *I = llvm::lower_bound(Table, MI.getOpcode());
275 if (I == Table.end() || I->EvexOpcode != MI.getOpcode())
276 return false;
277
278 unsigned NewOpc = I->VexOpcode;
279
280 if (usesExtendedRegister(MI))
281 return false;
282
283 if (!performCustomAdjustments(MI, NewOpc, ST))
284 return false;
285
286 MI.setDesc(TII->get(NewOpc));
287 MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
288 return true;
289 }
290
INITIALIZE_PASS(EvexToVexInstPass,EVEX2VEX_NAME,EVEX2VEX_DESC,false,false)291 INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false)
292
293 FunctionPass *llvm::createX86EvexToVexInsts() {
294 return new EvexToVexInstPass();
295 }
296