• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
10 #include "HexagonISelLowering.h"
11 #include "HexagonRegisterInfo.h"
12 #include "HexagonSubtarget.h"
13 #include "llvm/Support/CommandLine.h"
14 
15 using namespace llvm;
16 
17 static const MVT LegalV64[] =  { MVT::v64i8,  MVT::v32i16,  MVT::v16i32 };
18 static const MVT LegalW64[] =  { MVT::v128i8, MVT::v64i16,  MVT::v32i32 };
19 static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16,  MVT::v32i32 };
20 static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
21 
22 
23 void
initializeHVXLowering()24 HexagonTargetLowering::initializeHVXLowering() {
25   if (Subtarget.useHVX64BOps()) {
26     addRegisterClass(MVT::v64i8,  &Hexagon::HvxVRRegClass);
27     addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
28     addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
29     addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
30     addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
31     addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
32     // These "short" boolean vector types should be legal because
33     // they will appear as results of vector compares. If they were
34     // not legal, type legalization would try to make them legal
35     // and that would require using operations that do not use or
36     // produce such types. That, in turn, would imply using custom
37     // nodes, which would be unoptimizable by the DAG combiner.
38     // The idea is to rely on target-independent operations as much
39     // as possible.
40     addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
41     addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
42     addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
43     addRegisterClass(MVT::v512i1, &Hexagon::HvxQRRegClass);
44   } else if (Subtarget.useHVX128BOps()) {
45     addRegisterClass(MVT::v128i8,  &Hexagon::HvxVRRegClass);
46     addRegisterClass(MVT::v64i16,  &Hexagon::HvxVRRegClass);
47     addRegisterClass(MVT::v32i32,  &Hexagon::HvxVRRegClass);
48     addRegisterClass(MVT::v256i8,  &Hexagon::HvxWRRegClass);
49     addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
50     addRegisterClass(MVT::v64i32,  &Hexagon::HvxWRRegClass);
51     addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
52     addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
53     addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
54     addRegisterClass(MVT::v1024i1, &Hexagon::HvxQRRegClass);
55   }
56 
57   // Set up operation actions.
58 
59   bool Use64b = Subtarget.useHVX64BOps();
60   ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
61   ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
62   MVT ByteV = Use64b ?  MVT::v64i8 : MVT::v128i8;
63   MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
64 
65   auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
66     setOperationAction(Opc, FromTy, Promote);
67     AddPromotedToType(Opc, FromTy, ToTy);
68   };
69 
70   setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal);
71   setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal);
72 
73   for (MVT T : LegalV) {
74     setIndexedLoadAction(ISD::POST_INC,  T, Legal);
75     setIndexedStoreAction(ISD::POST_INC, T, Legal);
76 
77     setOperationAction(ISD::AND,            T, Legal);
78     setOperationAction(ISD::OR,             T, Legal);
79     setOperationAction(ISD::XOR,            T, Legal);
80     setOperationAction(ISD::ADD,            T, Legal);
81     setOperationAction(ISD::SUB,            T, Legal);
82     setOperationAction(ISD::CTPOP,          T, Legal);
83     setOperationAction(ISD::CTLZ,           T, Legal);
84     if (T != ByteV) {
85       setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
86       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
87       setOperationAction(ISD::BSWAP,                    T, Legal);
88     }
89 
90     setOperationAction(ISD::CTTZ,               T, Custom);
91     setOperationAction(ISD::LOAD,               T, Custom);
92     setOperationAction(ISD::MUL,                T, Custom);
93     setOperationAction(ISD::MULHS,              T, Custom);
94     setOperationAction(ISD::MULHU,              T, Custom);
95     setOperationAction(ISD::BUILD_VECTOR,       T, Custom);
96     // Make concat-vectors custom to handle concats of more than 2 vectors.
97     setOperationAction(ISD::CONCAT_VECTORS,     T, Custom);
98     setOperationAction(ISD::INSERT_SUBVECTOR,   T, Custom);
99     setOperationAction(ISD::INSERT_VECTOR_ELT,  T, Custom);
100     setOperationAction(ISD::EXTRACT_SUBVECTOR,  T, Custom);
101     setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
102     setOperationAction(ISD::ANY_EXTEND,         T, Custom);
103     setOperationAction(ISD::SIGN_EXTEND,        T, Custom);
104     setOperationAction(ISD::ZERO_EXTEND,        T, Custom);
105     if (T != ByteV) {
106       setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
107       // HVX only has shifts of words and halfwords.
108       setOperationAction(ISD::SRA,                     T, Custom);
109       setOperationAction(ISD::SHL,                     T, Custom);
110       setOperationAction(ISD::SRL,                     T, Custom);
111 
112       // Promote all shuffles to operate on vectors of bytes.
113       setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
114     }
115 
116     setCondCodeAction(ISD::SETNE,  T, Expand);
117     setCondCodeAction(ISD::SETLE,  T, Expand);
118     setCondCodeAction(ISD::SETGE,  T, Expand);
119     setCondCodeAction(ISD::SETLT,  T, Expand);
120     setCondCodeAction(ISD::SETULE, T, Expand);
121     setCondCodeAction(ISD::SETUGE, T, Expand);
122     setCondCodeAction(ISD::SETULT, T, Expand);
123   }
124 
125   for (MVT T : LegalW) {
126     // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
127     // independent) handling of it would convert it to a load, which is
128     // not always the optimal choice.
129     setOperationAction(ISD::BUILD_VECTOR,   T, Custom);
130     // Make concat-vectors custom to handle concats of more than 2 vectors.
131     setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
132 
133     // Custom-lower these operations for pairs. Expand them into a concat
134     // of the corresponding operations on individual vectors.
135     setOperationAction(ISD::ANY_EXTEND,               T, Custom);
136     setOperationAction(ISD::SIGN_EXTEND,              T, Custom);
137     setOperationAction(ISD::ZERO_EXTEND,              T, Custom);
138     setOperationAction(ISD::SIGN_EXTEND_INREG,        T, Custom);
139     setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG,  T, Custom);
140     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
141     setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
142 
143     setOperationAction(ISD::LOAD,     T, Custom);
144     setOperationAction(ISD::STORE,    T, Custom);
145     setOperationAction(ISD::CTLZ,     T, Custom);
146     setOperationAction(ISD::CTTZ,     T, Custom);
147     setOperationAction(ISD::CTPOP,    T, Custom);
148 
149     setOperationAction(ISD::ADD,      T, Legal);
150     setOperationAction(ISD::SUB,      T, Legal);
151     setOperationAction(ISD::MUL,      T, Custom);
152     setOperationAction(ISD::MULHS,    T, Custom);
153     setOperationAction(ISD::MULHU,    T, Custom);
154     setOperationAction(ISD::AND,      T, Custom);
155     setOperationAction(ISD::OR,       T, Custom);
156     setOperationAction(ISD::XOR,      T, Custom);
157     setOperationAction(ISD::SETCC,    T, Custom);
158     setOperationAction(ISD::VSELECT,  T, Custom);
159     if (T != ByteW) {
160       setOperationAction(ISD::SRA,      T, Custom);
161       setOperationAction(ISD::SHL,      T, Custom);
162       setOperationAction(ISD::SRL,      T, Custom);
163 
164       // Promote all shuffles to operate on vectors of bytes.
165       setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
166     }
167   }
168 
169   // Boolean vectors.
170 
171   for (MVT T : LegalW) {
172     // Boolean types for vector pairs will overlap with the boolean
173     // types for single vectors, e.g.
174     //   v64i8  -> v64i1 (single)
175     //   v64i16 -> v64i1 (pair)
176     // Set these actions first, and allow the single actions to overwrite
177     // any duplicates.
178     MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
179     setOperationAction(ISD::SETCC,              BoolW, Custom);
180     setOperationAction(ISD::AND,                BoolW, Custom);
181     setOperationAction(ISD::OR,                 BoolW, Custom);
182     setOperationAction(ISD::XOR,                BoolW, Custom);
183   }
184 
185   for (MVT T : LegalV) {
186     MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
187     setOperationAction(ISD::BUILD_VECTOR,       BoolV, Custom);
188     setOperationAction(ISD::CONCAT_VECTORS,     BoolV, Custom);
189     setOperationAction(ISD::INSERT_SUBVECTOR,   BoolV, Custom);
190     setOperationAction(ISD::INSERT_VECTOR_ELT,  BoolV, Custom);
191     setOperationAction(ISD::EXTRACT_SUBVECTOR,  BoolV, Custom);
192     setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom);
193     setOperationAction(ISD::AND,                BoolV, Legal);
194     setOperationAction(ISD::OR,                 BoolV, Legal);
195     setOperationAction(ISD::XOR,                BoolV, Legal);
196   }
197 }
198 
199 SDValue
getInt(unsigned IntId,MVT ResTy,ArrayRef<SDValue> Ops,const SDLoc & dl,SelectionDAG & DAG) const200 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
201                               const SDLoc &dl, SelectionDAG &DAG) const {
202   SmallVector<SDValue,4> IntOps;
203   IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
204   for (const SDValue &Op : Ops)
205     IntOps.push_back(Op);
206   return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
207 }
208 
209 MVT
typeJoin(const TypePair & Tys) const210 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
211   assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
212 
213   MVT ElemTy = Tys.first.getVectorElementType();
214   return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
215                                   Tys.second.getVectorNumElements());
216 }
217 
218 HexagonTargetLowering::TypePair
typeSplit(MVT VecTy) const219 HexagonTargetLowering::typeSplit(MVT VecTy) const {
220   assert(VecTy.isVector());
221   unsigned NumElem = VecTy.getVectorNumElements();
222   assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
223   MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
224   return { HalfTy, HalfTy };
225 }
226 
227 MVT
typeExtElem(MVT VecTy,unsigned Factor) const228 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
229   MVT ElemTy = VecTy.getVectorElementType();
230   MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
231   return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
232 }
233 
234 MVT
typeTruncElem(MVT VecTy,unsigned Factor) const235 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
236   MVT ElemTy = VecTy.getVectorElementType();
237   MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
238   return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
239 }
240 
241 SDValue
opCastElem(SDValue Vec,MVT ElemTy,SelectionDAG & DAG) const242 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
243                                   SelectionDAG &DAG) const {
244   if (ty(Vec).getVectorElementType() == ElemTy)
245     return Vec;
246   MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
247   return DAG.getBitcast(CastTy, Vec);
248 }
249 
250 SDValue
opJoin(const VectorPair & Ops,const SDLoc & dl,SelectionDAG & DAG) const251 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
252                               SelectionDAG &DAG) const {
253   return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
254                      Ops.second, Ops.first);
255 }
256 
257 HexagonTargetLowering::VectorPair
opSplit(SDValue Vec,const SDLoc & dl,SelectionDAG & DAG) const258 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
259                                SelectionDAG &DAG) const {
260   TypePair Tys = typeSplit(ty(Vec));
261   if (Vec.getOpcode() == HexagonISD::QCAT)
262     return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
263   return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
264 }
265 
266 bool
isHvxSingleTy(MVT Ty) const267 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
268   return Subtarget.isHVXVectorType(Ty) &&
269          Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
270 }
271 
272 bool
isHvxPairTy(MVT Ty) const273 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
274   return Subtarget.isHVXVectorType(Ty) &&
275          Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
276 }
277 
278 SDValue
convertToByteIndex(SDValue ElemIdx,MVT ElemTy,SelectionDAG & DAG) const279 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
280                                           SelectionDAG &DAG) const {
281   if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
282     ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
283 
284   unsigned ElemWidth = ElemTy.getSizeInBits();
285   if (ElemWidth == 8)
286     return ElemIdx;
287 
288   unsigned L = Log2_32(ElemWidth/8);
289   const SDLoc &dl(ElemIdx);
290   return DAG.getNode(ISD::SHL, dl, MVT::i32,
291                      {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
292 }
293 
294 SDValue
getIndexInWord32(SDValue Idx,MVT ElemTy,SelectionDAG & DAG) const295 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
296                                         SelectionDAG &DAG) const {
297   unsigned ElemWidth = ElemTy.getSizeInBits();
298   assert(ElemWidth >= 8 && ElemWidth <= 32);
299   if (ElemWidth == 32)
300     return Idx;
301 
302   if (ty(Idx) != MVT::i32)
303     Idx = DAG.getBitcast(MVT::i32, Idx);
304   const SDLoc &dl(Idx);
305   SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
306   SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
307   return SubIdx;
308 }
309 
310 SDValue
getByteShuffle(const SDLoc & dl,SDValue Op0,SDValue Op1,ArrayRef<int> Mask,SelectionDAG & DAG) const311 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
312                                       SDValue Op1, ArrayRef<int> Mask,
313                                       SelectionDAG &DAG) const {
314   MVT OpTy = ty(Op0);
315   assert(OpTy == ty(Op1));
316 
317   MVT ElemTy = OpTy.getVectorElementType();
318   if (ElemTy == MVT::i8)
319     return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
320   assert(ElemTy.getSizeInBits() >= 8);
321 
322   MVT ResTy = tyVector(OpTy, MVT::i8);
323   unsigned ElemSize = ElemTy.getSizeInBits() / 8;
324 
325   SmallVector<int,128> ByteMask;
326   for (int M : Mask) {
327     if (M < 0) {
328       for (unsigned I = 0; I != ElemSize; ++I)
329         ByteMask.push_back(-1);
330     } else {
331       int NewM = M*ElemSize;
332       for (unsigned I = 0; I != ElemSize; ++I)
333         ByteMask.push_back(NewM+I);
334     }
335   }
336   assert(ResTy.getVectorNumElements() == ByteMask.size());
337   return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
338                               opCastElem(Op1, MVT::i8, DAG), ByteMask);
339 }
340 
341 SDValue
buildHvxVectorReg(ArrayRef<SDValue> Values,const SDLoc & dl,MVT VecTy,SelectionDAG & DAG) const342 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
343                                          const SDLoc &dl, MVT VecTy,
344                                          SelectionDAG &DAG) const {
345   unsigned VecLen = Values.size();
346   MachineFunction &MF = DAG.getMachineFunction();
347   MVT ElemTy = VecTy.getVectorElementType();
348   unsigned ElemWidth = ElemTy.getSizeInBits();
349   unsigned HwLen = Subtarget.getVectorLength();
350 
351   unsigned ElemSize = ElemWidth / 8;
352   assert(ElemSize*VecLen == HwLen);
353   SmallVector<SDValue,32> Words;
354 
355   if (VecTy.getVectorElementType() != MVT::i32) {
356     assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
357     unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
358     MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
359     for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
360       SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
361       Words.push_back(DAG.getBitcast(MVT::i32, W));
362     }
363   } else {
364     Words.assign(Values.begin(), Values.end());
365   }
366 
367   unsigned NumWords = Words.size();
368   bool IsSplat = true, IsUndef = true;
369   SDValue SplatV;
370   for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
371     if (isUndef(Words[i]))
372       continue;
373     IsUndef = false;
374     if (!SplatV.getNode())
375       SplatV = Words[i];
376     else if (SplatV != Words[i])
377       IsSplat = false;
378   }
379   if (IsUndef)
380     return DAG.getUNDEF(VecTy);
381   if (IsSplat) {
382     assert(SplatV.getNode());
383     auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
384     if (IdxN && IdxN->isNullValue())
385       return getZero(dl, VecTy, DAG);
386     return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
387   }
388 
389   // Delay recognizing constant vectors until here, so that we can generate
390   // a vsplat.
391   SmallVector<ConstantInt*, 128> Consts(VecLen);
392   bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
393   if (AllConst) {
394     ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
395                             (Constant**)Consts.end());
396     Constant *CV = ConstantVector::get(Tmp);
397     unsigned Align = HwLen;
398     SDValue CP = LowerConstantPool(DAG.getConstantPool(CV, VecTy, Align), DAG);
399     return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
400                        MachinePointerInfo::getConstantPool(MF), Align);
401   }
402 
403   // Construct two halves in parallel, then or them together.
404   assert(4*Words.size() == Subtarget.getVectorLength());
405   SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
406   SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
407   SDValue S = DAG.getConstant(4, dl, MVT::i32);
408   for (unsigned i = 0; i != NumWords/2; ++i) {
409     SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
410                             {HalfV0, Words[i]});
411     SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
412                             {HalfV1, Words[i+NumWords/2]});
413     HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
414     HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
415   }
416 
417   HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
418                        {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
419   SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
420   return DstV;
421 }
422 
423 SDValue
createHvxPrefixPred(SDValue PredV,const SDLoc & dl,unsigned BitBytes,bool ZeroFill,SelectionDAG & DAG) const424 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
425       unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
426   MVT PredTy = ty(PredV);
427   unsigned HwLen = Subtarget.getVectorLength();
428   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
429 
430   if (Subtarget.isHVXVectorType(PredTy, true)) {
431     // Move the vector predicate SubV to a vector register, and scale it
432     // down to match the representation (bytes per type element) that VecV
433     // uses. The scaling down will pick every 2nd or 4th (every Scale-th
434     // in general) element and put them at the front of the resulting
435     // vector. This subvector will then be inserted into the Q2V of VecV.
436     // To avoid having an operation that generates an illegal type (short
437     // vector), generate a full size vector.
438     //
439     SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
440     SmallVector<int,128> Mask(HwLen);
441     // Scale = BitBytes(PredV) / Given BitBytes.
442     unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
443     unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
444 
445     for (unsigned i = 0; i != HwLen; ++i) {
446       unsigned Num = i % Scale;
447       unsigned Off = i / Scale;
448       Mask[BlockLen*Num + Off] = i;
449     }
450     SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
451     if (!ZeroFill)
452       return S;
453     // Fill the bytes beyond BlockLen with 0s.
454     MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
455     SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
456                          {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
457     SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
458     return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
459   }
460 
461   // Make sure that this is a valid scalar predicate.
462   assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
463 
464   unsigned Bytes = 8 / PredTy.getVectorNumElements();
465   SmallVector<SDValue,4> Words[2];
466   unsigned IdxW = 0;
467 
468   auto Lo32 = [&DAG, &dl] (SDValue P) {
469     return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
470   };
471   auto Hi32 = [&DAG, &dl] (SDValue P) {
472     return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
473   };
474 
475   SDValue W0 = isUndef(PredV)
476                   ? DAG.getUNDEF(MVT::i64)
477                   : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
478   Words[IdxW].push_back(Hi32(W0));
479   Words[IdxW].push_back(Lo32(W0));
480 
481   while (Bytes < BitBytes) {
482     IdxW ^= 1;
483     Words[IdxW].clear();
484 
485     if (Bytes < 4) {
486       for (const SDValue &W : Words[IdxW ^ 1]) {
487         SDValue T = expandPredicate(W, dl, DAG);
488         Words[IdxW].push_back(Hi32(T));
489         Words[IdxW].push_back(Lo32(T));
490       }
491     } else {
492       for (const SDValue &W : Words[IdxW ^ 1]) {
493         Words[IdxW].push_back(W);
494         Words[IdxW].push_back(W);
495       }
496     }
497     Bytes *= 2;
498   }
499 
500   assert(Bytes == BitBytes);
501 
502   SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
503   SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
504   for (const SDValue &W : Words[IdxW]) {
505     Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
506     Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
507   }
508 
509   return Vec;
510 }
511 
512 SDValue
buildHvxVectorPred(ArrayRef<SDValue> Values,const SDLoc & dl,MVT VecTy,SelectionDAG & DAG) const513 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
514                                           const SDLoc &dl, MVT VecTy,
515                                           SelectionDAG &DAG) const {
516   // Construct a vector V of bytes, such that a comparison V >u 0 would
517   // produce the required vector predicate.
518   unsigned VecLen = Values.size();
519   unsigned HwLen = Subtarget.getVectorLength();
520   assert(VecLen <= HwLen || VecLen == 8*HwLen);
521   SmallVector<SDValue,128> Bytes;
522   bool AllT = true, AllF = true;
523 
524   auto IsTrue = [] (SDValue V) {
525     if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
526       return !N->isNullValue();
527     return false;
528   };
529   auto IsFalse = [] (SDValue V) {
530     if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
531       return N->isNullValue();
532     return false;
533   };
534 
535   if (VecLen <= HwLen) {
536     // In the hardware, each bit of a vector predicate corresponds to a byte
537     // of a vector register. Calculate how many bytes does a bit of VecTy
538     // correspond to.
539     assert(HwLen % VecLen == 0);
540     unsigned BitBytes = HwLen / VecLen;
541     for (SDValue V : Values) {
542       AllT &= IsTrue(V);
543       AllF &= IsFalse(V);
544 
545       SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
546                                  : DAG.getUNDEF(MVT::i8);
547       for (unsigned B = 0; B != BitBytes; ++B)
548         Bytes.push_back(Ext);
549     }
550   } else {
551     // There are as many i1 values, as there are bits in a vector register.
552     // Divide the values into groups of 8 and check that each group consists
553     // of the same value (ignoring undefs).
554     for (unsigned I = 0; I != VecLen; I += 8) {
555       unsigned B = 0;
556       // Find the first non-undef value in this group.
557       for (; B != 8; ++B) {
558         if (!Values[I+B].isUndef())
559           break;
560       }
561       SDValue F = Values[I+B];
562       AllT &= IsTrue(F);
563       AllF &= IsFalse(F);
564 
565       SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
566                             : DAG.getUNDEF(MVT::i8);
567       Bytes.push_back(Ext);
568       // Verify that the rest of values in the group are the same as the
569       // first.
570       for (; B != 8; ++B)
571         assert(Values[I+B].isUndef() || Values[I+B] == F);
572     }
573   }
574 
575   if (AllT)
576     return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
577   if (AllF)
578     return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
579 
580   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
581   SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
582   return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
583 }
584 
585 SDValue
extractHvxElementReg(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const586 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
587       const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
588   MVT ElemTy = ty(VecV).getVectorElementType();
589 
590   unsigned ElemWidth = ElemTy.getSizeInBits();
591   assert(ElemWidth >= 8 && ElemWidth <= 32);
592   (void)ElemWidth;
593 
594   SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
595   SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
596                                {VecV, ByteIdx});
597   if (ElemTy == MVT::i32)
598     return ExWord;
599 
600   // Have an extracted word, need to extract the smaller element out of it.
601   // 1. Extract the bits of (the original) IdxV that correspond to the index
602   //    of the desired element in the 32-bit word.
603   SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
604   // 2. Extract the element from the word.
605   SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
606   return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
607 }
608 
609 SDValue
extractHvxElementPred(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const610 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
611       const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
612   // Implement other return types if necessary.
613   assert(ResTy == MVT::i1);
614 
615   unsigned HwLen = Subtarget.getVectorLength();
616   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
617   SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
618 
619   unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
620   SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
621   IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
622 
623   SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
624   SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
625   return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
626 }
627 
628 SDValue
insertHvxElementReg(SDValue VecV,SDValue IdxV,SDValue ValV,const SDLoc & dl,SelectionDAG & DAG) const629 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
630       SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
631   MVT ElemTy = ty(VecV).getVectorElementType();
632 
633   unsigned ElemWidth = ElemTy.getSizeInBits();
634   assert(ElemWidth >= 8 && ElemWidth <= 32);
635   (void)ElemWidth;
636 
637   auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
638                                      SDValue ByteIdxV) {
639     MVT VecTy = ty(VecV);
640     unsigned HwLen = Subtarget.getVectorLength();
641     SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
642                                 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
643     SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
644     SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
645     SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
646                                {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
647     SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
648     return TorV;
649   };
650 
651   SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
652   if (ElemTy == MVT::i32)
653     return InsertWord(VecV, ValV, ByteIdx);
654 
655   // If this is not inserting a 32-bit word, convert it into such a thing.
656   // 1. Extract the existing word from the target vector.
657   SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
658                                 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
659   SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
660                                      dl, MVT::i32, DAG);
661 
662   // 2. Treating the extracted word as a 32-bit vector, insert the given
663   //    value into it.
664   SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
665   MVT SubVecTy = tyVector(ty(Ext), ElemTy);
666   SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
667                              ValV, SubIdx, dl, ElemTy, DAG);
668 
669   // 3. Insert the 32-bit word back into the original vector.
670   return InsertWord(VecV, Ins, ByteIdx);
671 }
672 
673 SDValue
insertHvxElementPred(SDValue VecV,SDValue IdxV,SDValue ValV,const SDLoc & dl,SelectionDAG & DAG) const674 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
675       SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
676   unsigned HwLen = Subtarget.getVectorLength();
677   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
678   SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
679 
680   unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
681   SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
682   IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
683   ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
684 
685   SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
686   return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
687 }
688 
689 SDValue
extractHvxSubvectorReg(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const690 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
691       const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
692   MVT VecTy = ty(VecV);
693   unsigned HwLen = Subtarget.getVectorLength();
694   unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
695   MVT ElemTy = VecTy.getVectorElementType();
696   unsigned ElemWidth = ElemTy.getSizeInBits();
697 
698   // If the source vector is a vector pair, get the single vector containing
699   // the subvector of interest. The subvector will never overlap two single
700   // vectors.
701   if (isHvxPairTy(VecTy)) {
702     unsigned SubIdx;
703     if (Idx * ElemWidth >= 8*HwLen) {
704       SubIdx = Hexagon::vsub_hi;
705       Idx -= VecTy.getVectorNumElements() / 2;
706     } else {
707       SubIdx = Hexagon::vsub_lo;
708     }
709     VecTy = typeSplit(VecTy).first;
710     VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
711     if (VecTy == ResTy)
712       return VecV;
713   }
714 
715   // The only meaningful subvectors of a single HVX vector are those that
716   // fit in a scalar register.
717   assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
718 
719   MVT WordTy = tyVector(VecTy, MVT::i32);
720   SDValue WordVec = DAG.getBitcast(WordTy, VecV);
721   unsigned WordIdx = (Idx*ElemWidth) / 32;
722 
723   SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
724   SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
725   if (ResTy.getSizeInBits() == 32)
726     return DAG.getBitcast(ResTy, W0);
727 
728   SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
729   SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
730   SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
731   return DAG.getBitcast(ResTy, WW);
732 }
733 
734 SDValue
extractHvxSubvectorPred(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const735 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
736       const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
737   MVT VecTy = ty(VecV);
738   unsigned HwLen = Subtarget.getVectorLength();
739   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
740   SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
741   // IdxV is required to be a constant.
742   unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
743 
744   unsigned ResLen = ResTy.getVectorNumElements();
745   unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
746   unsigned Offset = Idx * BitBytes;
747   SDValue Undef = DAG.getUNDEF(ByteTy);
748   SmallVector<int,128> Mask;
749 
750   if (Subtarget.isHVXVectorType(ResTy, true)) {
751     // Converting between two vector predicates. Since the result is shorter
752     // than the source, it will correspond to a vector predicate with the
753     // relevant bits replicated. The replication count is the ratio of the
754     // source and target vector lengths.
755     unsigned Rep = VecTy.getVectorNumElements() / ResLen;
756     assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
757     for (unsigned i = 0; i != HwLen/Rep; ++i) {
758       for (unsigned j = 0; j != Rep; ++j)
759         Mask.push_back(i + Offset);
760     }
761     SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
762     return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
763   }
764 
765   // Converting between a vector predicate and a scalar predicate. In the
766   // vector predicate, a group of BitBytes bits will correspond to a single
767   // i1 element of the source vector type. Those bits will all have the same
768   // value. The same will be true for ByteVec, where each byte corresponds
769   // to a bit in the vector predicate.
770   // The algorithm is to traverse the ByteVec, going over the i1 values from
771   // the source vector, and generate the corresponding representation in an
772   // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
773   // elements so that the interesting 8 bytes will be in the low end of the
774   // vector.
775   unsigned Rep = 8 / ResLen;
776   // Make sure the output fill the entire vector register, so repeat the
777   // 8-byte groups as many times as necessary.
778   for (unsigned r = 0; r != HwLen/ResLen; ++r) {
779     // This will generate the indexes of the 8 interesting bytes.
780     for (unsigned i = 0; i != ResLen; ++i) {
781       for (unsigned j = 0; j != Rep; ++j)
782         Mask.push_back(Offset + i*BitBytes);
783     }
784   }
785 
786   SDValue Zero = getZero(dl, MVT::i32, DAG);
787   SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
788   // Combine the two low words from ShuffV into a v8i8, and byte-compare
789   // them against 0.
790   SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
791   SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
792                            {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
793   SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
794   return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
795                   {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
796 }
797 
798 SDValue
insertHvxSubvectorReg(SDValue VecV,SDValue SubV,SDValue IdxV,const SDLoc & dl,SelectionDAG & DAG) const799 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
800       SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
801   MVT VecTy = ty(VecV);
802   MVT SubTy = ty(SubV);
803   unsigned HwLen = Subtarget.getVectorLength();
804   MVT ElemTy = VecTy.getVectorElementType();
805   unsigned ElemWidth = ElemTy.getSizeInBits();
806 
807   bool IsPair = isHvxPairTy(VecTy);
808   MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
809   // The two single vectors that VecV consists of, if it's a pair.
810   SDValue V0, V1;
811   SDValue SingleV = VecV;
812   SDValue PickHi;
813 
814   if (IsPair) {
815     V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
816     V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
817 
818     SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
819                                     dl, MVT::i32);
820     PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
821     if (isHvxSingleTy(SubTy)) {
822       if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
823         unsigned Idx = CN->getZExtValue();
824         assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
825         unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
826         return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
827       }
828       // If IdxV is not a constant, generate the two variants: with the
829       // SubV as the high and as the low subregister, and select the right
830       // pair based on the IdxV.
831       SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
832       SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
833       return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
834     }
835     // The subvector being inserted must be entirely contained in one of
836     // the vectors V0 or V1. Set SingleV to the correct one, and update
837     // IdxV to be the index relative to the beginning of that vector.
838     SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
839     IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
840     SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
841   }
842 
843   // The only meaningful subvectors of a single HVX vector are those that
844   // fit in a scalar register.
845   assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
846   // Convert IdxV to be index in bytes.
847   auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
848   if (!IdxN || !IdxN->isNullValue()) {
849     IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
850                        DAG.getConstant(ElemWidth/8, dl, MVT::i32));
851     SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
852   }
853   // When inserting a single word, the rotation back to the original position
854   // would be by HwLen-Idx, but if two words are inserted, it will need to be
855   // by (HwLen-4)-Idx.
856   unsigned RolBase = HwLen;
857   if (VecTy.getSizeInBits() == 32) {
858     SDValue V = DAG.getBitcast(MVT::i32, SubV);
859     SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
860   } else {
861     SDValue V = DAG.getBitcast(MVT::i64, SubV);
862     SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
863     SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
864     SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
865     SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
866                           DAG.getConstant(4, dl, MVT::i32));
867     SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
868     RolBase = HwLen-4;
869   }
870   // If the vector wasn't ror'ed, don't ror it back.
871   if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
872     SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
873                                DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
874     SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
875   }
876 
877   if (IsPair) {
878     SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
879     SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
880     return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
881   }
882   return SingleV;
883 }
884 
885 SDValue
insertHvxSubvectorPred(SDValue VecV,SDValue SubV,SDValue IdxV,const SDLoc & dl,SelectionDAG & DAG) const886 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
887       SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
888   MVT VecTy = ty(VecV);
889   MVT SubTy = ty(SubV);
890   assert(Subtarget.isHVXVectorType(VecTy, true));
891   // VecV is an HVX vector predicate. SubV may be either an HVX vector
892   // predicate as well, or it can be a scalar predicate.
893 
894   unsigned VecLen = VecTy.getVectorNumElements();
895   unsigned HwLen = Subtarget.getVectorLength();
896   assert(HwLen % VecLen == 0 && "Unexpected vector type");
897 
898   unsigned Scale = VecLen / SubTy.getVectorNumElements();
899   unsigned BitBytes = HwLen / VecLen;
900   unsigned BlockLen = HwLen / Scale;
901 
902   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
903   SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
904   SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
905   SDValue ByteIdx;
906 
907   auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
908   if (!IdxN || !IdxN->isNullValue()) {
909     ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
910                           DAG.getConstant(BitBytes, dl, MVT::i32));
911     ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
912   }
913 
914   // ByteVec is the target vector VecV rotated in such a way that the
915   // subvector should be inserted at index 0. Generate a predicate mask
916   // and use vmux to do the insertion.
917   MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
918   SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
919                        {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
920   ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
921   // Rotate ByteVec back, and convert to a vector predicate.
922   if (!IdxN || !IdxN->isNullValue()) {
923     SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
924     SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
925     ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
926   }
927   return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
928 }
929 
930 SDValue
extendHvxVectorPred(SDValue VecV,const SDLoc & dl,MVT ResTy,bool ZeroExt,SelectionDAG & DAG) const931 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
932       MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
933   // Sign- and any-extending of a vector predicate to a vector register is
934   // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
935   // a vector of 1s (where the 1s are of type matching the vector type).
936   assert(Subtarget.isHVXVectorType(ResTy));
937   if (!ZeroExt)
938     return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
939 
940   assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
941   SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
942                              DAG.getConstant(1, dl, MVT::i32));
943   SDValue False = getZero(dl, ResTy, DAG);
944   return DAG.getSelect(dl, ResTy, VecV, True, False);
945 }
946 
947 SDValue
LowerHvxBuildVector(SDValue Op,SelectionDAG & DAG) const948 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
949       const {
950   const SDLoc &dl(Op);
951   MVT VecTy = ty(Op);
952 
953   unsigned Size = Op.getNumOperands();
954   SmallVector<SDValue,128> Ops;
955   for (unsigned i = 0; i != Size; ++i)
956     Ops.push_back(Op.getOperand(i));
957 
958   if (VecTy.getVectorElementType() == MVT::i1)
959     return buildHvxVectorPred(Ops, dl, VecTy, DAG);
960 
961   if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
962     ArrayRef<SDValue> A(Ops);
963     MVT SingleTy = typeSplit(VecTy).first;
964     SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
965     SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
966     return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
967   }
968 
969   return buildHvxVectorReg(Ops, dl, VecTy, DAG);
970 }
971 
972 SDValue
LowerHvxConcatVectors(SDValue Op,SelectionDAG & DAG) const973 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
974       const {
975   // Vector concatenation of two integer (non-bool) vectors does not need
976   // special lowering. Custom-lower concats of bool vectors and expand
977   // concats of more than 2 vectors.
978   MVT VecTy = ty(Op);
979   const SDLoc &dl(Op);
980   unsigned NumOp = Op.getNumOperands();
981   if (VecTy.getVectorElementType() != MVT::i1) {
982     if (NumOp == 2)
983       return Op;
984     // Expand the other cases into a build-vector.
985     SmallVector<SDValue,8> Elems;
986     for (SDValue V : Op.getNode()->ops())
987       DAG.ExtractVectorElements(V, Elems);
988     // A vector of i16 will be broken up into a build_vector of i16's.
989     // This is a problem, since at the time of operation legalization,
990     // all operations are expected to be type-legalized, and i16 is not
991     // a legal type. If any of the extracted elements is not of a valid
992     // type, sign-extend it to a valid one.
993     for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
994       SDValue V = Elems[i];
995       MVT Ty = ty(V);
996       if (!isTypeLegal(Ty)) {
997         EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
998         if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
999           Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1000                                  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1001                                              V.getOperand(0), V.getOperand(1)),
1002                                  DAG.getValueType(Ty));
1003           continue;
1004         }
1005         // A few less complicated cases.
1006         if (V.getOpcode() == ISD::Constant)
1007           Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1008         else if (V.isUndef())
1009           Elems[i] = DAG.getUNDEF(NTy);
1010         else
1011           llvm_unreachable("Unexpected vector element");
1012       }
1013     }
1014     return DAG.getBuildVector(VecTy, dl, Elems);
1015   }
1016 
1017   assert(VecTy.getVectorElementType() == MVT::i1);
1018   unsigned HwLen = Subtarget.getVectorLength();
1019   assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1020 
1021   SDValue Op0 = Op.getOperand(0);
1022 
1023   // If the operands are HVX types (i.e. not scalar predicates), then
1024   // defer the concatenation, and create QCAT instead.
1025   if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1026     if (NumOp == 2)
1027       return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1028 
1029     ArrayRef<SDUse> U(Op.getNode()->ops());
1030     SmallVector<SDValue,4> SV(U.begin(), U.end());
1031     ArrayRef<SDValue> Ops(SV);
1032 
1033     MVT HalfTy = typeSplit(VecTy).first;
1034     SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1035                              Ops.take_front(NumOp/2));
1036     SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1037                              Ops.take_back(NumOp/2));
1038     return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1039   }
1040 
1041   // Count how many bytes (in a vector register) each bit in VecTy
1042   // corresponds to.
1043   unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1044 
1045   SmallVector<SDValue,8> Prefixes;
1046   for (SDValue V : Op.getNode()->op_values()) {
1047     SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1048     Prefixes.push_back(P);
1049   }
1050 
1051   unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1052   MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1053   SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1054   SDValue Res = getZero(dl, ByteTy, DAG);
1055   for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1056     Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1057     Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1058   }
1059   return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1060 }
1061 
1062 SDValue
LowerHvxExtractElement(SDValue Op,SelectionDAG & DAG) const1063 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1064       const {
1065   // Change the type of the extracted element to i32.
1066   SDValue VecV = Op.getOperand(0);
1067   MVT ElemTy = ty(VecV).getVectorElementType();
1068   const SDLoc &dl(Op);
1069   SDValue IdxV = Op.getOperand(1);
1070   if (ElemTy == MVT::i1)
1071     return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1072 
1073   return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1074 }
1075 
1076 SDValue
LowerHvxInsertElement(SDValue Op,SelectionDAG & DAG) const1077 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1078       const {
1079   const SDLoc &dl(Op);
1080   SDValue VecV = Op.getOperand(0);
1081   SDValue ValV = Op.getOperand(1);
1082   SDValue IdxV = Op.getOperand(2);
1083   MVT ElemTy = ty(VecV).getVectorElementType();
1084   if (ElemTy == MVT::i1)
1085     return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1086 
1087   return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1088 }
1089 
1090 SDValue
LowerHvxExtractSubvector(SDValue Op,SelectionDAG & DAG) const1091 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1092       const {
1093   SDValue SrcV = Op.getOperand(0);
1094   MVT SrcTy = ty(SrcV);
1095   MVT DstTy = ty(Op);
1096   SDValue IdxV = Op.getOperand(1);
1097   unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1098   assert(Idx % DstTy.getVectorNumElements() == 0);
1099   (void)Idx;
1100   const SDLoc &dl(Op);
1101 
1102   MVT ElemTy = SrcTy.getVectorElementType();
1103   if (ElemTy == MVT::i1)
1104     return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1105 
1106   return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1107 }
1108 
1109 SDValue
LowerHvxInsertSubvector(SDValue Op,SelectionDAG & DAG) const1110 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1111       const {
1112   // Idx does not need to be a constant.
1113   SDValue VecV = Op.getOperand(0);
1114   SDValue ValV = Op.getOperand(1);
1115   SDValue IdxV = Op.getOperand(2);
1116 
1117   const SDLoc &dl(Op);
1118   MVT VecTy = ty(VecV);
1119   MVT ElemTy = VecTy.getVectorElementType();
1120   if (ElemTy == MVT::i1)
1121     return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1122 
1123   return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1124 }
1125 
1126 SDValue
LowerHvxAnyExt(SDValue Op,SelectionDAG & DAG) const1127 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1128   // Lower any-extends of boolean vectors to sign-extends, since they
1129   // translate directly to Q2V. Zero-extending could also be done equally
1130   // fast, but Q2V is used/recognized in more places.
1131   // For all other vectors, use zero-extend.
1132   MVT ResTy = ty(Op);
1133   SDValue InpV = Op.getOperand(0);
1134   MVT ElemTy = ty(InpV).getVectorElementType();
1135   if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1136     return LowerHvxSignExt(Op, DAG);
1137   return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1138 }
1139 
1140 SDValue
LowerHvxSignExt(SDValue Op,SelectionDAG & DAG) const1141 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1142   MVT ResTy = ty(Op);
1143   SDValue InpV = Op.getOperand(0);
1144   MVT ElemTy = ty(InpV).getVectorElementType();
1145   if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1146     return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1147   return Op;
1148 }
1149 
1150 SDValue
LowerHvxZeroExt(SDValue Op,SelectionDAG & DAG) const1151 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1152   MVT ResTy = ty(Op);
1153   SDValue InpV = Op.getOperand(0);
1154   MVT ElemTy = ty(InpV).getVectorElementType();
1155   if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1156     return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1157   return Op;
1158 }
1159 
1160 SDValue
LowerHvxCttz(SDValue Op,SelectionDAG & DAG) const1161 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1162   // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1163   // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1164   const SDLoc &dl(Op);
1165   MVT ResTy = ty(Op);
1166   SDValue InpV = Op.getOperand(0);
1167   assert(ResTy == ty(InpV));
1168 
1169   // Calculate the vectors of 1 and bitwidth(x).
1170   MVT ElemTy = ty(InpV).getVectorElementType();
1171   unsigned ElemWidth = ElemTy.getSizeInBits();
1172   // Using uint64_t because a shift by 32 can happen.
1173   uint64_t Splat1 = 0, SplatW = 0;
1174   assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1175   for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1176     Splat1 = (Splat1 << ElemWidth) | 1;
1177     SplatW = (SplatW << ElemWidth) | ElemWidth;
1178   }
1179   SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1180                              DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1181   SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1182                              DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1183   SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1184                               DAG.getConstant(-1, dl, MVT::i32));
1185   // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1186   // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1187   // it separately in custom combine or selection).
1188   SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1189                           {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1190                            DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1191   return DAG.getNode(ISD::SUB, dl, ResTy,
1192                      {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1193 }
1194 
1195 SDValue
LowerHvxMul(SDValue Op,SelectionDAG & DAG) const1196 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1197   MVT ResTy = ty(Op);
1198   assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1199   const SDLoc &dl(Op);
1200   SmallVector<int,256> ShuffMask;
1201 
1202   MVT ElemTy = ResTy.getVectorElementType();
1203   unsigned VecLen = ResTy.getVectorNumElements();
1204   SDValue Vs = Op.getOperand(0);
1205   SDValue Vt = Op.getOperand(1);
1206 
1207   switch (ElemTy.SimpleTy) {
1208     case MVT::i8: {
1209       // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1210       // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1211       // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1212       MVT ExtTy = typeExtElem(ResTy, 2);
1213       unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1214                                           : Hexagon::V6_vmpyhv;
1215       SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1216 
1217       // Discard high halves of the resulting values, collect the low halves.
1218       for (unsigned I = 0; I < VecLen; I += 2) {
1219         ShuffMask.push_back(I);         // Pick even element.
1220         ShuffMask.push_back(I+VecLen);  // Pick odd element.
1221       }
1222       VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1223       SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1224       return DAG.getBitcast(ResTy, BS);
1225     }
1226     case MVT::i16:
1227       // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1228       // (There is also V6_vmpyhv, which behaves in an analogous way to
1229       // V6_vmpybv.)
1230       return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1231     case MVT::i32: {
1232       // Use the following sequence for signed word multiply:
1233       // T0 = V6_vmpyiowh Vs, Vt
1234       // T1 = V6_vaslw T0, 16
1235       // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1236       SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1237       SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1238       SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1239       SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1240                             {T1, Vs, Vt}, DAG);
1241       return T2;
1242     }
1243     default:
1244       break;
1245   }
1246   return SDValue();
1247 }
1248 
1249 SDValue
LowerHvxMulh(SDValue Op,SelectionDAG & DAG) const1250 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1251   MVT ResTy = ty(Op);
1252   assert(ResTy.isVector());
1253   const SDLoc &dl(Op);
1254   SmallVector<int,256> ShuffMask;
1255 
1256   MVT ElemTy = ResTy.getVectorElementType();
1257   unsigned VecLen = ResTy.getVectorNumElements();
1258   SDValue Vs = Op.getOperand(0);
1259   SDValue Vt = Op.getOperand(1);
1260   bool IsSigned = Op.getOpcode() == ISD::MULHS;
1261 
1262   if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1263     // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1264     // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1265     // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1266     // For i16, use V6_vmpyhv, which behaves in an analogous way to
1267     // V6_vmpybv: results Lo and Hi are products of even/odd elements
1268     // respectively.
1269     MVT ExtTy = typeExtElem(ResTy, 2);
1270     unsigned MpyOpc = ElemTy == MVT::i8
1271         ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1272         : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1273     SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1274 
1275     // Discard low halves of the resulting values, collect the high halves.
1276     for (unsigned I = 0; I < VecLen; I += 2) {
1277       ShuffMask.push_back(I+1);         // Pick even element.
1278       ShuffMask.push_back(I+VecLen+1);  // Pick odd element.
1279     }
1280     VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1281     SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1282     return DAG.getBitcast(ResTy, BS);
1283   }
1284 
1285   assert(ElemTy == MVT::i32);
1286   SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1287 
1288   if (IsSigned) {
1289     // mulhs(Vs,Vt) =
1290     //   = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1291     //   = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1292     //      + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1293     //   = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1294     //      + Lo(Vs) *us Vt] >> 32
1295     // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1296     // anything, so it cannot produce any carry over to higher bits),
1297     // so everything in [] can be shifted by 16 without loss of precision.
1298     //   = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1299     //   = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1300     // Denote Hi(Vs) = Vs':
1301     //   = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1302     //   = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1303     SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1304     // Get Vs':
1305     SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1306     SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1307                           {T0, S0, Vt}, DAG);
1308     // Shift by 16:
1309     SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1310     // Get Vs'*Hi(Vt):
1311     SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1312     // Add:
1313     SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1314     return T3;
1315   }
1316 
1317   // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1318 
1319   auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1320     return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1321   };
1322   auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1323     return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1324   };
1325 
1326   MVT PairTy = typeJoin({ResTy, ResTy});
1327   SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1328                        {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1329   // Multiply-unsigned halfwords:
1330   //   LoVec = Vs.uh[2i] * Vt.uh[2i],
1331   //   HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1332   SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1333   // The low halves in the LoVec of the pair can be discarded. They are
1334   // not added to anything (in the full-precision product), so they cannot
1335   // produce a carry into the higher bits.
1336   SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1337   // Swap low and high halves in Vt, and do the halfword multiplication
1338   // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1339   SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1340   SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1341   // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1342   // These products are words, but cannot be added directly because the
1343   // sums could overflow. Add these products, by halfwords, where each sum
1344   // of a pair of halfwords gives a word.
1345   SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1346                         {LoVec(T2), HiVec(T2)}, DAG);
1347   // Add the high halfwords from the products of the low halfwords.
1348   SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1349   SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1350   SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1351   SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1352   return T7;
1353 }
1354 
1355 SDValue
LowerHvxExtend(SDValue Op,SelectionDAG & DAG) const1356 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1357   // Sign- and zero-extends are legal.
1358   assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
1359   return DAG.getZeroExtendVectorInReg(Op.getOperand(0), SDLoc(Op), ty(Op));
1360 }
1361 
1362 SDValue
LowerHvxShift(SDValue Op,SelectionDAG & DAG) const1363 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1364   if (SDValue S = getVectorShiftByInt(Op, DAG))
1365     return S;
1366   return Op;
1367 }
1368 
1369 SDValue
SplitHvxPairOp(SDValue Op,SelectionDAG & DAG) const1370 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1371   assert(!Op.isMachineOpcode());
1372   SmallVector<SDValue,2> OpsL, OpsH;
1373   const SDLoc &dl(Op);
1374 
1375   auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1376     MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1377     SDValue TV = DAG.getValueType(Ty);
1378     return std::make_pair(TV, TV);
1379   };
1380 
1381   for (SDValue A : Op.getNode()->ops()) {
1382     VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1383                     ? opSplit(A, dl, DAG)
1384                     : std::make_pair(A, A);
1385     // Special case for type operand.
1386     if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1387       if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1388         P = SplitVTNode(N);
1389     }
1390     OpsL.push_back(P.first);
1391     OpsH.push_back(P.second);
1392   }
1393 
1394   MVT ResTy = ty(Op);
1395   MVT HalfTy = typeSplit(ResTy).first;
1396   SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1397   SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1398   SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1399   return S;
1400 }
1401 
1402 SDValue
SplitHvxMemOp(SDValue Op,SelectionDAG & DAG) const1403 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1404   LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1405   assert(BN->isUnindexed());
1406   MVT MemTy = BN->getMemoryVT().getSimpleVT();
1407   if (!isHvxPairTy(MemTy))
1408     return Op;
1409 
1410   const SDLoc &dl(Op);
1411   unsigned HwLen = Subtarget.getVectorLength();
1412   MVT SingleTy = typeSplit(MemTy).first;
1413   SDValue Chain = BN->getChain();
1414   SDValue Base0 = BN->getBasePtr();
1415   SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1416 
1417   MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1418   if (MachineMemOperand *MMO = BN->getMemOperand()) {
1419     MachineFunction &MF = DAG.getMachineFunction();
1420     MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1421     MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1422   }
1423 
1424   unsigned MemOpc = BN->getOpcode();
1425   SDValue NewOp;
1426 
1427   if (MemOpc == ISD::LOAD) {
1428     SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1429     SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1430     NewOp = DAG.getMergeValues(
1431               { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1432                 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1433                             Load0.getValue(1), Load1.getValue(1)) }, dl);
1434   } else {
1435     assert(MemOpc == ISD::STORE);
1436     VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1437     SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1438     SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1439     NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1440   }
1441 
1442   return NewOp;
1443 }
1444 
1445 SDValue
LowerHvxOperation(SDValue Op,SelectionDAG & DAG) const1446 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1447   unsigned Opc = Op.getOpcode();
1448   bool IsPairOp = isHvxPairTy(ty(Op)) ||
1449                   llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1450                     return isHvxPairTy(ty(V));
1451                   });
1452 
1453   if (IsPairOp) {
1454     switch (Opc) {
1455       default:
1456         break;
1457       case ISD::LOAD:
1458       case ISD::STORE:
1459         return SplitHvxMemOp(Op, DAG);
1460       case ISD::CTPOP:
1461       case ISD::CTLZ:
1462       case ISD::CTTZ:
1463       case ISD::MUL:
1464       case ISD::MULHS:
1465       case ISD::MULHU:
1466       case ISD::AND:
1467       case ISD::OR:
1468       case ISD::XOR:
1469       case ISD::SRA:
1470       case ISD::SHL:
1471       case ISD::SRL:
1472       case ISD::SETCC:
1473       case ISD::VSELECT:
1474       case ISD::SIGN_EXTEND_INREG:
1475         return SplitHvxPairOp(Op, DAG);
1476     }
1477   }
1478 
1479   switch (Opc) {
1480     default:
1481       break;
1482     case ISD::BUILD_VECTOR:            return LowerHvxBuildVector(Op, DAG);
1483     case ISD::CONCAT_VECTORS:          return LowerHvxConcatVectors(Op, DAG);
1484     case ISD::INSERT_SUBVECTOR:        return LowerHvxInsertSubvector(Op, DAG);
1485     case ISD::INSERT_VECTOR_ELT:       return LowerHvxInsertElement(Op, DAG);
1486     case ISD::EXTRACT_SUBVECTOR:       return LowerHvxExtractSubvector(Op, DAG);
1487     case ISD::EXTRACT_VECTOR_ELT:      return LowerHvxExtractElement(Op, DAG);
1488 
1489     case ISD::ANY_EXTEND:              return LowerHvxAnyExt(Op, DAG);
1490     case ISD::SIGN_EXTEND:             return LowerHvxSignExt(Op, DAG);
1491     case ISD::ZERO_EXTEND:             return LowerHvxZeroExt(Op, DAG);
1492     case ISD::CTTZ:                    return LowerHvxCttz(Op, DAG);
1493     case ISD::SRA:
1494     case ISD::SHL:
1495     case ISD::SRL:                     return LowerHvxShift(Op, DAG);
1496     case ISD::MUL:                     return LowerHvxMul(Op, DAG);
1497     case ISD::MULHS:
1498     case ISD::MULHU:                   return LowerHvxMulh(Op, DAG);
1499     case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1500     case ISD::SETCC:
1501     case ISD::INTRINSIC_VOID:          return Op;
1502     // Unaligned loads will be handled by the default lowering.
1503     case ISD::LOAD:                    return SDValue();
1504   }
1505 #ifndef NDEBUG
1506   Op.dumpr(&DAG);
1507 #endif
1508   llvm_unreachable("Unhandled HVX operation");
1509 }
1510 
1511 bool
isHvxOperation(SDValue Op) const1512 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1513   // If the type of the result, or any operand type are HVX vector types,
1514   // this is an HVX operation.
1515   return Subtarget.isHVXVectorType(ty(Op), true) ||
1516          llvm::any_of(Op.getNode()->ops(),
1517                       [this] (SDValue V) {
1518                         return Subtarget.isHVXVectorType(ty(V), true);
1519                       });
1520 }
1521