• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1//===- NVPTXVector.td - NVPTX Vector Specific Instruction defs -*- tblgen-*-==//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10//-----------------------------------
11// Vector Specific
12//-----------------------------------
13
14//
15// All vector instructions derive from NVPTXVecInst
16//
17
18class NVPTXVecInst<dag outs, dag ins, string asmstr, list<dag> pattern,
19  NVPTXInst sInst=NOP>
20  : NVPTXInst<outs, ins, asmstr, pattern> {
21  NVPTXInst scalarInst=sInst;
22}
23
24let isAsCheapAsAMove=1, VecInstType=isVecExtract.Value in {
25// Extract v2i16
26def V2i16Extract : NVPTXVecInst<(outs Int16Regs:$dst),
27  (ins V2I16Regs:$src, i8imm:$c),
28                         "mov.u16 \t$dst, $src${c:vecelem};",
29                         [(set Int16Regs:$dst, (vector_extract
30                           (v2i16 V2I16Regs:$src), imm:$c))],
31                         IMOV16rr>;
32
33// Extract v4i16
34def V4i16Extract : NVPTXVecInst<(outs Int16Regs:$dst),
35  (ins V4I16Regs:$src, i8imm:$c),
36                         "mov.u16 \t$dst, $src${c:vecelem};",
37                         [(set Int16Regs:$dst, (vector_extract
38                           (v4i16 V4I16Regs:$src), imm:$c))],
39                         IMOV16rr>;
40
41// Extract v2i8
42def V2i8Extract : NVPTXVecInst<(outs Int8Regs:$dst),
43  (ins V2I8Regs:$src, i8imm:$c),
44                         "mov.u16 \t$dst, $src${c:vecelem};",
45                         [(set Int8Regs:$dst, (vector_extract
46                           (v2i8 V2I8Regs:$src), imm:$c))],
47                         IMOV8rr>;
48
49// Extract v4i8
50def V4i8Extract : NVPTXVecInst<(outs Int8Regs:$dst),
51  (ins V4I8Regs:$src, i8imm:$c),
52                         "mov.u16 \t$dst, $src${c:vecelem};",
53                         [(set Int8Regs:$dst, (vector_extract
54                           (v4i8 V4I8Regs:$src), imm:$c))],
55                         IMOV8rr>;
56
57// Extract v2i32
58def V2i32Extract : NVPTXVecInst<(outs Int32Regs:$dst),
59  (ins V2I32Regs:$src, i8imm:$c),
60                         "mov.u32 \t$dst, $src${c:vecelem};",
61                         [(set Int32Regs:$dst, (vector_extract
62                           (v2i32 V2I32Regs:$src), imm:$c))],
63                         IMOV32rr>;
64
65// Extract v2f32
66def V2f32Extract : NVPTXVecInst<(outs Float32Regs:$dst),
67  (ins V2F32Regs:$src, i8imm:$c),
68                         "mov.f32 \t$dst, $src${c:vecelem};",
69                         [(set Float32Regs:$dst, (vector_extract
70                           (v2f32 V2F32Regs:$src), imm:$c))],
71                         FMOV32rr>;
72
73// Extract v2i64
74def V2i64Extract : NVPTXVecInst<(outs Int64Regs:$dst),
75  (ins V2I64Regs:$src, i8imm:$c),
76                         "mov.u64 \t$dst, $src${c:vecelem};",
77                         [(set Int64Regs:$dst, (vector_extract
78                           (v2i64 V2I64Regs:$src), imm:$c))],
79                         IMOV64rr>;
80
81// Extract v2f64
82def V2f64Extract : NVPTXVecInst<(outs Float64Regs:$dst),
83  (ins V2F64Regs:$src, i8imm:$c),
84                         "mov.f64 \t$dst, $src${c:vecelem};",
85                         [(set Float64Regs:$dst, (vector_extract
86                           (v2f64 V2F64Regs:$src), imm:$c))],
87                         FMOV64rr>;
88
89// Extract v4i32
90def V4i32Extract : NVPTXVecInst<(outs Int32Regs:$dst),
91  (ins V4I32Regs:$src, i8imm:$c),
92                         "mov.u32 \t$dst, $src${c:vecelem};",
93                         [(set Int32Regs:$dst, (vector_extract
94                           (v4i32 V4I32Regs:$src), imm:$c))],
95                         IMOV32rr>;
96
97// Extract v4f32
98def V4f32Extract : NVPTXVecInst<(outs Float32Regs:$dst),
99  (ins V4F32Regs:$src, i8imm:$c),
100                         "mov.f32 \t$dst, $src${c:vecelem};",
101                         [(set Float32Regs:$dst, (vector_extract
102                           (v4f32 V4F32Regs:$src), imm:$c))],
103                         FMOV32rr>;
104}
105
106let isAsCheapAsAMove=1, VecInstType=isVecInsert.Value in {
107// Insert v2i8
108def V2i8Insert : NVPTXVecInst<(outs V2I8Regs:$dst),
109  (ins V2I8Regs:$src, Int8Regs:$val, i8imm:$c),
110        "mov.v2.u16 \t${dst:vecfull}, ${src:vecfull};"
111        "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
112       [(set V2I8Regs:$dst,
113         (vector_insert V2I8Regs:$src, Int8Regs:$val, imm:$c))],
114                         IMOV8rr>;
115
116// Insert v4i8
117def V4i8Insert : NVPTXVecInst<(outs V4I8Regs:$dst),
118  (ins V4I8Regs:$src, Int8Regs:$val, i8imm:$c),
119                       "mov.v4.u16 \t${dst:vecfull}, ${src:vecfull};"
120                       "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
121       [(set V4I8Regs:$dst,
122         (vector_insert V4I8Regs:$src, Int8Regs:$val, imm:$c))],
123                         IMOV8rr>;
124
125// Insert v2i16
126def V2i16Insert : NVPTXVecInst<(outs V2I16Regs:$dst),
127  (ins V2I16Regs:$src, Int16Regs:$val, i8imm:$c),
128                       "mov.v2.u16 \t${dst:vecfull}, ${src:vecfull};"
129                       "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
130       [(set V2I16Regs:$dst,
131         (vector_insert V2I16Regs:$src, Int16Regs:$val, imm:$c))],
132                         IMOV16rr>;
133
134// Insert v4i16
135def V4i16Insert : NVPTXVecInst<(outs V4I16Regs:$dst),
136  (ins V4I16Regs:$src, Int16Regs:$val, i8imm:$c),
137                       "mov.v4.u16 \t${dst:vecfull}, ${src:vecfull};"
138                       "\n\tmov.u16 \t$dst${c:vecelem}, $val;",
139       [(set V4I16Regs:$dst,
140         (vector_insert V4I16Regs:$src, Int16Regs:$val, imm:$c))],
141                         IMOV16rr>;
142
143// Insert v2i32
144def V2i32Insert : NVPTXVecInst<(outs V2I32Regs:$dst),
145  (ins V2I32Regs:$src, Int32Regs:$val, i8imm:$c),
146                       "mov.v2.u32 \t${dst:vecfull}, ${src:vecfull};"
147                       "\n\tmov.u32 \t$dst${c:vecelem}, $val;",
148       [(set V2I32Regs:$dst,
149         (vector_insert V2I32Regs:$src, Int32Regs:$val, imm:$c))],
150                         IMOV32rr>;
151
152// Insert v2f32
153def V2f32Insert : NVPTXVecInst<(outs V2F32Regs:$dst),
154  (ins V2F32Regs:$src, Float32Regs:$val, i8imm:$c),
155                       "mov.v2.f32 \t${dst:vecfull}, ${src:vecfull};"
156                       "\n\tmov.f32 \t$dst${c:vecelem}, $val;",
157       [(set V2F32Regs:$dst,
158         (vector_insert V2F32Regs:$src, Float32Regs:$val, imm:$c))],
159                         FMOV32rr>;
160
161// Insert v2i64
162def V2i64Insert : NVPTXVecInst<(outs V2I64Regs:$dst),
163  (ins V2I64Regs:$src, Int64Regs:$val, i8imm:$c),
164                       "mov.v2.u64 \t${dst:vecfull}, ${src:vecfull};"
165                       "\n\tmov.u64 \t$dst${c:vecelem}, $val;",
166       [(set V2I64Regs:$dst,
167         (vector_insert V2I64Regs:$src, Int64Regs:$val, imm:$c))],
168                         IMOV64rr>;
169
170// Insert v2f64
171def V2f64Insert : NVPTXVecInst<(outs V2F64Regs:$dst),
172  (ins V2F64Regs:$src, Float64Regs:$val, i8imm:$c),
173                       "mov.v2.f64 \t${dst:vecfull}, ${src:vecfull};"
174                       "\n\tmov.f64 \t$dst${c:vecelem}, $val;",
175       [(set V2F64Regs:$dst,
176         (vector_insert V2F64Regs:$src, Float64Regs:$val, imm:$c))],
177                         FMOV64rr>;
178
179// Insert v4i32
180def V4i32Insert : NVPTXVecInst<(outs V4I32Regs:$dst),
181  (ins V4I32Regs:$src, Int32Regs:$val, i8imm:$c),
182                       "mov.v4.u32 \t${dst:vecfull}, ${src:vecfull};"
183                       "\n\tmov.u32 \t$dst${c:vecelem}, $val;",
184       [(set V4I32Regs:$dst,
185         (vector_insert V4I32Regs:$src, Int32Regs:$val, imm:$c))],
186                         IMOV32rr>;
187
188// Insert v4f32
189def V4f32Insert : NVPTXVecInst<(outs V4F32Regs:$dst),
190  (ins V4F32Regs:$src, Float32Regs:$val, i8imm:$c),
191                       "mov.v4.f32 \t${dst:vecfull}, ${src:vecfull};"
192                       "\n\tmov.f32 \t$dst${c:vecelem}, $val;",
193       [(set V4F32Regs:$dst,
194         (vector_insert V4F32Regs:$src, Float32Regs:$val, imm:$c))],
195                         FMOV32rr>;
196}
197
198class BinOpAsmString<string c> {
199  string s = c;
200}
201
202class V4AsmStr<string opcode> : BinOpAsmString<
203                          !strconcat(!strconcat(!strconcat(!strconcat(
204                            !strconcat(!strconcat(!strconcat(
205                          opcode,  " \t${dst}_0, ${a}_0, ${b}_0;\n\t"),
206                          opcode), " \t${dst}_1, ${a}_1, ${b}_1;\n\t"),
207                          opcode), " \t${dst}_2, ${a}_2, ${b}_2;\n\t"),
208                          opcode), " \t${dst}_3, ${a}_3, ${b}_3;")>;
209
210class V2AsmStr<string opcode> : BinOpAsmString<
211                           !strconcat(!strconcat(!strconcat(
212                           opcode,  " \t${dst}_0, ${a}_0, ${b}_0;\n\t"),
213                           opcode), " \t${dst}_1, ${a}_1, ${b}_1;")>;
214
215class V4MADStr<string opcode> : BinOpAsmString<
216                          !strconcat(!strconcat(!strconcat(!strconcat(
217                            !strconcat(!strconcat(!strconcat(
218                          opcode,  " \t${dst}_0, ${a}_0, ${b}_0, ${c}_0;\n\t"),
219                          opcode), " \t${dst}_1, ${a}_1, ${b}_1, ${c}_1;\n\t"),
220                          opcode), " \t${dst}_2, ${a}_2, ${b}_2, ${c}_2;\n\t"),
221                          opcode), " \t${dst}_3, ${a}_3, ${b}_3, ${c}_3;")>;
222
223class V2MADStr<string opcode> : BinOpAsmString<
224                           !strconcat(!strconcat(!strconcat(
225                           opcode,  " \t${dst}_0, ${a}_0, ${b}_0, ${c}_0;\n\t"),
226                           opcode), " \t${dst}_1, ${a}_1, ${b}_1, ${c}_1;")>;
227
228class V4UnaryStr<string opcode> : BinOpAsmString<
229                          !strconcat(!strconcat(!strconcat(!strconcat(
230                            !strconcat(!strconcat(!strconcat(
231                          opcode,  " \t${dst}_0, ${a}_0;\n\t"),
232                          opcode), " \t${dst}_1, ${a}_1;\n\t"),
233                          opcode), " \t${dst}_2, ${a}_2;\n\t"),
234                          opcode), " \t${dst}_3, ${a}_3;")>;
235
236class V2UnaryStr<string opcode> : BinOpAsmString<
237                           !strconcat(!strconcat(!strconcat(
238                           opcode,  " \t${dst}_0, ${a}_0;\n\t"),
239                           opcode), " \t${dst}_1, ${a}_1;")>;
240
241class VecBinaryOp<BinOpAsmString asmstr, SDNode OpNode, NVPTXRegClass regclass,
242  NVPTXInst sInst=NOP> :
243      NVPTXVecInst<(outs regclass:$dst), (ins regclass:$a, regclass:$b),
244                 asmstr.s,
245                 [(set regclass:$dst, (OpNode regclass:$a, regclass:$b))],
246                 sInst>;
247
248class VecShiftOp<BinOpAsmString asmstr, SDNode OpNode, NVPTXRegClass regclass1,
249                 NVPTXRegClass regclass2, NVPTXInst sInst=NOP> :
250      NVPTXVecInst<(outs regclass1:$dst), (ins regclass1:$a, regclass2:$b),
251                 asmstr.s,
252                 [(set regclass1:$dst, (OpNode regclass1:$a, regclass2:$b))],
253                 sInst>;
254
255class VecUnaryOp<BinOpAsmString asmstr, PatFrag OpNode, NVPTXRegClass regclass,
256  NVPTXInst sInst=NOP> :
257      NVPTXVecInst<(outs regclass:$dst), (ins regclass:$a),
258                 asmstr.s,
259                 [(set regclass:$dst, (OpNode regclass:$a))], sInst>;
260
261multiclass IntBinVOp<string asmstr, SDNode OpNode,
262                     NVPTXInst i64op=NOP, NVPTXInst i32op=NOP, NVPTXInst
263                     i16op=NOP, NVPTXInst i8op=NOP> {
264  def V2I64 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "64")>, OpNode, V2I64Regs,
265    i64op>;
266  def V4I32 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "32")>, OpNode, V4I32Regs,
267    i32op>;
268  def V2I32 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "32")>, OpNode, V2I32Regs,
269    i32op>;
270  def V4I16 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "16")>, OpNode, V4I16Regs,
271    i16op>;
272  def V2I16 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "16")>, OpNode, V2I16Regs,
273    i16op>;
274  def V4I8 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "16")>, OpNode, V4I8Regs,
275    i8op>;
276  def V2I8 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "16")>, OpNode, V2I8Regs,
277    i8op>;
278}
279
280multiclass FloatBinVOp<string asmstr, SDNode OpNode,
281                       NVPTXInst f64=NOP, NVPTXInst f32=NOP,
282                       NVPTXInst f32_ftz=NOP> {
283  def V2F64 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "f64")>, OpNode,
284    V2F64Regs, f64>;
285  def V4F32_ftz : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "ftz.f32")>, OpNode,
286    V4F32Regs, f32_ftz>, Requires<[doF32FTZ]>;
287  def V2F32_ftz : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "ftz.f32")>, OpNode,
288    V2F32Regs, f32_ftz>, Requires<[doF32FTZ]>;
289  def V4F32 : VecBinaryOp<V4AsmStr<!strconcat(asmstr, "f32")>, OpNode,
290    V4F32Regs, f32>;
291  def V2F32 : VecBinaryOp<V2AsmStr<!strconcat(asmstr, "f32")>, OpNode,
292    V2F32Regs, f32>;
293}
294
295multiclass IntUnaryVOp<string asmstr, PatFrag OpNode,
296                       NVPTXInst i64op=NOP, NVPTXInst i32op=NOP,
297                       NVPTXInst i16op=NOP, NVPTXInst i8op=NOP> {
298  def V2I64 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "64")>, OpNode,
299    V2I64Regs, i64op>;
300  def V4I32 : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "32")>, OpNode,
301    V4I32Regs, i32op>;
302  def V2I32 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "32")>, OpNode,
303    V2I32Regs, i32op>;
304  def V4I16 : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "16")>, OpNode,
305    V4I16Regs, i16op>;
306  def V2I16 : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "16")>, OpNode,
307    V2I16Regs, i16op>;
308  def V4I8  : VecUnaryOp<V4UnaryStr<!strconcat(asmstr, "16")>, OpNode,
309    V4I8Regs,   i8op>;
310  def V2I8  : VecUnaryOp<V2UnaryStr<!strconcat(asmstr, "16")>, OpNode,
311    V2I8Regs,   i8op>;
312}
313
314
315// Integer Arithmetic
316let VecInstType=isVecOther.Value in {
317defm VAdd : IntBinVOp<"add.s", add, ADDi64rr, ADDi32rr, ADDi16rr, ADDi8rr>;
318defm VSub : IntBinVOp<"sub.s", sub, SUBi64rr, SUBi32rr, SUBi16rr, SUBi8rr>;
319
320def AddCCV4I32 : VecBinaryOp<V4AsmStr<"add.cc.s32">, addc, V4I32Regs,
321  ADDCCi32rr>;
322def AddCCV2I32 : VecBinaryOp<V2AsmStr<"add.cc.s32">, addc, V2I32Regs,
323  ADDCCi32rr>;
324def SubCCV4I32 : VecBinaryOp<V4AsmStr<"sub.cc.s32">, subc, V4I32Regs,
325  SUBCCi32rr>;
326def SubCCV2I32 : VecBinaryOp<V2AsmStr<"sub.cc.s32">, subc, V2I32Regs,
327  SUBCCi32rr>;
328def AddCCCV4I32 : VecBinaryOp<V4AsmStr<"addc.cc.s32">, adde, V4I32Regs,
329  ADDCCCi32rr>;
330def AddCCCV2I32 : VecBinaryOp<V2AsmStr<"addc.cc.s32">, adde, V2I32Regs,
331  ADDCCCi32rr>;
332def SubCCCV4I32 : VecBinaryOp<V4AsmStr<"subc.cc.s32">, sube, V4I32Regs,
333  SUBCCCi32rr>;
334def SubCCCV2I32 : VecBinaryOp<V2AsmStr<"subc.cc.s32">, sube, V2I32Regs,
335  SUBCCCi32rr>;
336
337def ShiftLV2I64 : VecShiftOp<V2AsmStr<"shl.b64">, shl, V2I64Regs, V2I32Regs,
338  SHLi64rr>;
339def ShiftLV2I32 : VecShiftOp<V2AsmStr<"shl.b32">, shl, V2I32Regs, V2I32Regs,
340  SHLi32rr>;
341def ShiftLV4I32 : VecShiftOp<V4AsmStr<"shl.b32">, shl, V4I32Regs, V4I32Regs,
342  SHLi32rr>;
343def ShiftLV2I16 : VecShiftOp<V2AsmStr<"shl.b16">, shl, V2I16Regs, V2I32Regs,
344  SHLi16rr>;
345def ShiftLV4I16 : VecShiftOp<V4AsmStr<"shl.b16">, shl, V4I16Regs, V4I32Regs,
346  SHLi16rr>;
347def ShiftLV2I8  : VecShiftOp<V2AsmStr<"shl.b16">, shl, V2I8Regs,  V2I32Regs,
348  SHLi8rr>;
349def ShiftLV4I8  : VecShiftOp<V4AsmStr<"shl.b16">, shl, V4I8Regs,  V4I32Regs,
350  SHLi8rr>;
351}
352
353// cvt to v*i32, helpers for shift
354class CVTtoVeci32<NVPTXRegClass inclass, NVPTXRegClass outclass, string asmstr,
355  NVPTXInst sInst=NOP> :
356      NVPTXVecInst<(outs outclass:$d), (ins inclass:$s), asmstr, [], sInst>;
357
358class VecCVTStrHelper<string op, string dest, string src> {
359  string s=!strconcat(op, !strconcat("\t",
360           !strconcat(dest, !strconcat(", ", !strconcat(src, ";")))));
361}
362
363class Vec2CVTStr<string op> {
364  string s=!strconcat(VecCVTStrHelper<op, "${d}_0", "${s}_0">.s,
365           !strconcat("\n\t", VecCVTStrHelper<op, "${d}_1", "${s}_1">.s));
366}
367
368class Vec4CVTStr<string op> {
369  string s=!strconcat(VecCVTStrHelper<op, "${d}_0", "${s}_0">.s,
370           !strconcat("\n\t",
371           !strconcat(VecCVTStrHelper<op, "${d}_1", "${s}_1">.s,
372           !strconcat("\n\t",
373           !strconcat(VecCVTStrHelper<op, "${d}_2", "${s}_2">.s,
374           !strconcat("\n\t", VecCVTStrHelper<op, "${d}_3", "${s}_3">.s))))));
375}
376
377let VecInstType=isVecOther.Value in {
378def CVTv2i8tov2i32 : CVTtoVeci32<V2I8Regs, V2I32Regs,
379  Vec2CVTStr<"cvt.u32.u16">.s, Zint_extendext8to32>;
380def CVTv2i16tov2i32 : CVTtoVeci32<V2I16Regs, V2I32Regs,
381  Vec2CVTStr<"cvt.u32.u16">.s, Zint_extendext16to32>;
382def CVTv4i8tov4i32 : CVTtoVeci32<V4I8Regs, V4I32Regs,
383  Vec4CVTStr<"cvt.u32.u16">.s, Zint_extendext8to32>;
384def CVTv4i16tov4i32 : CVTtoVeci32<V4I16Regs, V4I32Regs,
385  Vec4CVTStr<"cvt.u32.u16">.s, Zint_extendext16to32>;
386def CVTv2i64tov2i32 : CVTtoVeci32<V2I64Regs, V2I32Regs,
387  Vec2CVTStr<"cvt.u32.u64">.s, TRUNC_64to32>;
388}
389
390def : Pat<(shl V2I16Regs:$src1, V2I16Regs:$src2),
391          (ShiftLV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
392def : Pat<(shl V2I8Regs:$src1, V2I8Regs:$src2),
393          (ShiftLV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
394def : Pat<(shl V2I64Regs:$src1, V2I64Regs:$src2),
395          (ShiftLV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
396
397def : Pat<(shl V4I16Regs:$src1, V4I16Regs:$src2),
398          (ShiftLV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
399def : Pat<(shl V4I8Regs:$src1, V4I8Regs:$src2),
400          (ShiftLV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
401
402let VecInstType=isVecOther.Value in {
403def ShiftRAV2I64 : VecShiftOp<V2AsmStr<"shr.s64">, sra, V2I64Regs, V2I32Regs,
404  SRAi64rr>;
405def ShiftRAV2I32 : VecShiftOp<V2AsmStr<"shr.s32">, sra, V2I32Regs, V2I32Regs,
406  SRAi32rr>;
407def ShiftRAV4I32 : VecShiftOp<V4AsmStr<"shr.s32">, sra, V4I32Regs, V4I32Regs,
408  SRAi32rr>;
409def ShiftRAV2I16 : VecShiftOp<V2AsmStr<"shr.s16">, sra, V2I16Regs, V2I32Regs,
410  SRAi16rr>;
411def ShiftRAV4I16 : VecShiftOp<V4AsmStr<"shr.s16">, sra, V4I16Regs, V4I32Regs,
412  SRAi16rr>;
413def ShiftRAV2I8  : VecShiftOp<V2AsmStr<"shr.s16">, sra, V2I8Regs,  V2I32Regs,
414  SRAi8rr>;
415def ShiftRAV4I8  : VecShiftOp<V4AsmStr<"shr.s16">, sra, V4I8Regs,  V4I32Regs,
416  SRAi8rr>;
417
418def ShiftRLV2I64 : VecShiftOp<V2AsmStr<"shr.u64">, srl, V2I64Regs, V2I32Regs,
419  SRLi64rr>;
420def ShiftRLV2I32 : VecShiftOp<V2AsmStr<"shr.u32">, srl, V2I32Regs, V2I32Regs,
421  SRLi32rr>;
422def ShiftRLV4I32 : VecShiftOp<V4AsmStr<"shr.u32">, srl, V4I32Regs, V4I32Regs,
423  SRLi32rr>;
424def ShiftRLV2I16 : VecShiftOp<V2AsmStr<"shr.u16">, srl, V2I16Regs, V2I32Regs,
425  SRLi16rr>;
426def ShiftRLV4I16 : VecShiftOp<V4AsmStr<"shr.u16">, srl, V4I16Regs, V4I32Regs,
427  SRLi16rr>;
428def ShiftRLV2I8  : VecShiftOp<V2AsmStr<"shr.u16">, srl, V2I8Regs,  V2I32Regs,
429  SRLi8rr>;
430def ShiftRLV4I8  : VecShiftOp<V4AsmStr<"shr.u16">, srl, V4I8Regs,  V4I32Regs,
431  SRLi8rr>;
432
433defm VMult   : IntBinVOp<"mul.lo.s", mul, MULTi64rr, MULTi32rr, MULTi16rr,
434  MULTi8rr>;
435defm VMultHS : IntBinVOp<"mul.hi.s", mulhs, MULTHSi64rr, MULTHSi32rr,
436  MULTHSi16rr,
437  MULTHSi8rr>;
438defm VMultHU : IntBinVOp<"mul.hi.u", mulhu, MULTHUi64rr, MULTHUi32rr,
439  MULTHUi16rr,
440  MULTHUi8rr>;
441defm VSDiv   : IntBinVOp<"div.s", sdiv, SDIVi64rr, SDIVi32rr, SDIVi16rr,
442  SDIVi8rr>;
443defm VUDiv   : IntBinVOp<"div.u", udiv, UDIVi64rr, UDIVi32rr, UDIVi16rr,
444  UDIVi8rr>;
445defm VSRem   : IntBinVOp<"rem.s", srem, SREMi64rr, SREMi32rr, SREMi16rr,
446  SREMi8rr>;
447defm VURem   : IntBinVOp<"rem.u", urem, UREMi64rr, UREMi32rr, UREMi16rr,
448  UREMi8rr>;
449}
450
451def : Pat<(sra V2I16Regs:$src1, V2I16Regs:$src2),
452          (ShiftRAV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
453def : Pat<(sra V2I8Regs:$src1, V2I8Regs:$src2),
454          (ShiftRAV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
455def : Pat<(sra V2I64Regs:$src1, V2I64Regs:$src2),
456          (ShiftRAV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
457
458def : Pat<(sra V4I16Regs:$src1, V4I16Regs:$src2),
459          (ShiftRAV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
460def : Pat<(sra V4I8Regs:$src1, V4I8Regs:$src2),
461          (ShiftRAV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
462
463def : Pat<(srl V2I16Regs:$src1, V2I16Regs:$src2),
464          (ShiftRLV2I16 V2I16Regs:$src1, (CVTv2i16tov2i32 V2I16Regs:$src2))>;
465def : Pat<(srl V2I8Regs:$src1, V2I8Regs:$src2),
466          (ShiftRLV2I8 V2I8Regs:$src1, (CVTv2i8tov2i32 V2I8Regs:$src2))>;
467def : Pat<(srl V2I64Regs:$src1, V2I64Regs:$src2),
468          (ShiftRLV2I64 V2I64Regs:$src1, (CVTv2i64tov2i32 V2I64Regs:$src2))>;
469
470def : Pat<(srl V4I16Regs:$src1, V4I16Regs:$src2),
471          (ShiftRLV4I16 V4I16Regs:$src1, (CVTv4i16tov4i32 V4I16Regs:$src2))>;
472def : Pat<(srl V4I8Regs:$src1, V4I8Regs:$src2),
473          (ShiftRLV4I8 V4I8Regs:$src1, (CVTv4i8tov4i32 V4I8Regs:$src2))>;
474
475multiclass VMAD<string asmstr, NVPTXRegClass regclassv4,
476  NVPTXRegClass regclassv2,
477                SDNode an=add, SDNode mn=mul, NVPTXInst sop=NOP,
478                Predicate Pred> {
479  def V4 : NVPTXVecInst<(outs regclassv4:$dst),
480    (ins regclassv4:$a, regclassv4:$b, regclassv4:$c),
481                      V4MADStr<asmstr>.s,
482                      [(set regclassv4:$dst,
483                        (an (mn regclassv4:$a, regclassv4:$b), regclassv4:$c))],
484                      sop>,
485           Requires<[Pred]>;
486  def V2 : NVPTXVecInst<(outs regclassv2:$dst),
487    (ins regclassv2:$a, regclassv2:$b, regclassv2:$c),
488                      V2MADStr<asmstr>.s,
489                      [(set regclassv2:$dst,
490                        (an (mn regclassv2:$a, regclassv2:$b), regclassv2:$c))],
491                      sop>,
492           Requires<[Pred]>;
493}
494
495multiclass VMADV2Only<string asmstr, NVPTXRegClass regclass, NVPTXInst sop=NOP,
496  Predicate Pred> {
497  def V2 : NVPTXVecInst<(outs regclass:$dst),
498    (ins regclass:$a, regclass:$b, regclass:$c),
499                      V2MADStr<asmstr>.s,
500                      [(set regclass:$dst, (add
501                        (mul regclass:$a, regclass:$b), regclass:$c))], sop>,
502           Requires<[Pred]>;
503}
504multiclass VFMADV2Only<string asmstr, NVPTXRegClass regclass, NVPTXInst sop=NOP,
505  Predicate Pred> {
506  def V2 : NVPTXVecInst<(outs regclass:$dst),
507    (ins regclass:$a, regclass:$b, regclass:$c),
508                      V2MADStr<asmstr>.s,
509                      [(set regclass:$dst, (fadd
510                        (fmul regclass:$a, regclass:$b), regclass:$c))], sop>,
511           Requires<[Pred]>;
512}
513
514let VecInstType=isVecOther.Value in {
515defm I8MAD  : VMAD<"mad.lo.s16", V4I8Regs, V2I8Regs, add, mul, MAD8rrr, true>;
516defm I16MAD : VMAD<"mad.lo.s16", V4I16Regs, V2I16Regs, add, mul, MAD16rrr,
517  true>;
518defm I32MAD : VMAD<"mad.lo.s32", V4I32Regs, V2I32Regs, add, mul, MAD32rrr,
519  true>;
520defm I64MAD : VMADV2Only<"mad.lo.s64", V2I64Regs, MAD64rrr, true>;
521
522defm VNeg : IntUnaryVOp<"neg.s", ineg, INEG64, INEG32, INEG16, INEG8>;
523
524defm VAddf : FloatBinVOp<"add.", fadd, FADDf64rr, FADDf32rr, FADDf32rr_ftz>;
525defm VSubf : FloatBinVOp<"sub.", fsub, FSUBf64rr, FSUBf32rr, FSUBf32rr_ftz>;
526defm VMulf : FloatBinVOp<"mul.", fmul, FMULf64rr, FMULf32rr, FMULf32rr_ftz>;
527
528defm F32MAD_ftz : VMAD<"mad.ftz.f32", V4F32Regs, V2F32Regs, fadd, fmul,
529  FMAD32_ftzrrr, doFMADF32_ftz>;
530defm F32FMA_ftz : VMAD<"fma.rn.ftz.f32", V4F32Regs, V2F32Regs, fadd, fmul,
531  FMA32_ftzrrr, doFMAF32_ftz>;
532defm F32MAD : VMAD<"mad.f32", V4F32Regs, V2F32Regs, fadd, fmul, FMAD32rrr,
533  doFMADF32>;
534defm F32FMA : VMAD<"fma.rn.f32", V4F32Regs, V2F32Regs, fadd, fmul, FMA32rrr,
535  doFMAF32>;
536defm F64FMA : VFMADV2Only<"fma.rn.f64", V2F64Regs, FMA64rrr, doFMAF64>;
537}
538
539let VecInstType=isVecOther.Value in {
540def V4F32Div_prec_ftz : VecBinaryOp<V4AsmStr<"div.rn.ftz.f32">, fdiv, V4F32Regs,
541  FDIV32rr_prec_ftz>, Requires<[doF32FTZ, reqPTX20]>;
542def V2F32Div_prec_ftz : VecBinaryOp<V2AsmStr<"div.rn.ftz.f32">, fdiv, V2F32Regs,
543  FDIV32rr_prec_ftz>, Requires<[doF32FTZ, reqPTX20]>;
544def V4F32Div_prec : VecBinaryOp<V4AsmStr<"div.rn.f32">, fdiv, V4F32Regs,
545  FDIV32rr_prec>, Requires<[reqPTX20]>;
546def V2F32Div_prec : VecBinaryOp<V2AsmStr<"div.rn.f32">, fdiv, V2F32Regs,
547  FDIV32rr_prec>, Requires<[reqPTX20]>;
548def V2F32Div_ftz : VecBinaryOp<V2AsmStr<"div.full.ftz.f32">, fdiv, V2F32Regs,
549  FDIV32rr_ftz>, Requires<[doF32FTZ]>;
550def V4F32Div_ftz : VecBinaryOp<V4AsmStr<"div.full.ftz.f32">, fdiv, V4F32Regs,
551  FDIV32rr_ftz>, Requires<[doF32FTZ]>;
552def V2F32Div : VecBinaryOp<V2AsmStr<"div.full.f32">, fdiv, V2F32Regs, FDIV32rr>;
553def V4F32Div : VecBinaryOp<V4AsmStr<"div.full.f32">, fdiv, V4F32Regs, FDIV32rr>;
554def V2F64Div : VecBinaryOp<V2AsmStr<"div.rn.f64">, fdiv, V2F64Regs, FDIV64rr>;
555}
556
557def fnegpat : PatFrag<(ops node:$in), (fneg node:$in)>;
558
559let VecInstType=isVecOther.Value in {
560def VNegv2f32_ftz : VecUnaryOp<V2UnaryStr<"neg.ftz.f32">, fnegpat, V2F32Regs,
561  FNEGf32_ftz>, Requires<[doF32FTZ]>;
562def VNegv4f32_ftz : VecUnaryOp<V4UnaryStr<"neg.ftz.f32">, fnegpat, V4F32Regs,
563  FNEGf32_ftz>, Requires<[doF32FTZ]>;
564def VNegv2f32 : VecUnaryOp<V2UnaryStr<"neg.f32">, fnegpat, V2F32Regs, FNEGf32>;
565def VNegv4f32 : VecUnaryOp<V4UnaryStr<"neg.f32">, fnegpat, V4F32Regs, FNEGf32>;
566def VNegv2f64 : VecUnaryOp<V2UnaryStr<"neg.f64">, fnegpat, V2F64Regs, FNEGf64>;
567
568// Logical Arithmetic
569defm VAnd : IntBinVOp<"and.b", and, ANDb64rr, ANDb32rr, ANDb16rr, ANDb8rr>;
570defm VOr  : IntBinVOp<"or.b", or, ORb64rr, ORb32rr, ORb16rr, ORb8rr>;
571defm VXor : IntBinVOp<"xor.b", xor, XORb64rr, XORb32rr, XORb16rr, XORb8rr>;
572
573defm VNot : IntUnaryVOp<"not.b", not, NOT64, NOT32, NOT16, NOT8>;
574}
575
576
577multiclass V2FPCONTRACT32_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
578  def : Pat<(fsub V2F32Regs:$a, (fmul V2F32Regs:$b, V2F32Regs:$c)),
579          (Inst (VNegv2f32 V2F32Regs:$b), V2F32Regs:$c,  V2F32Regs:$a)>,
580          Requires<[Pred]>;
581
582  def : Pat<(fsub (fmul V2F32Regs:$a, V2F32Regs:$b), V2F32Regs:$c),
583          (Inst V2F32Regs:$a, V2F32Regs:$b, (VNegv2f32 V2F32Regs:$c))>,
584          Requires<[Pred]>;
585}
586
587defm V2FMAF32ext_ftz  : V2FPCONTRACT32_SUB_PAT<F32FMA_ftzV2, doFMAF32AGG_ftz>;
588defm V2FMADF32ext_ftz : V2FPCONTRACT32_SUB_PAT<F32MAD_ftzV2, doFMADF32_ftz>;
589defm V2FMAF32ext  : V2FPCONTRACT32_SUB_PAT<F32FMAV2, doFMAF32AGG>;
590defm V2FMADF32ext : V2FPCONTRACT32_SUB_PAT<F32MADV2, doFMADF32>;
591
592multiclass V4FPCONTRACT32_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
593  def : Pat<(fsub V4F32Regs:$a, (fmul V4F32Regs:$b, V4F32Regs:$c)),
594          (Inst (VNegv4f32 V4F32Regs:$b), V4F32Regs:$c,  V4F32Regs:$a)>,
595          Requires<[Pred]>;
596
597  def : Pat<(fsub (fmul V4F32Regs:$a, V4F32Regs:$b), V4F32Regs:$c),
598          (Inst V4F32Regs:$a, V4F32Regs:$b, (VNegv4f32 V4F32Regs:$c))>,
599          Requires<[Pred]>;
600}
601
602defm V4FMAF32ext_ftz  : V4FPCONTRACT32_SUB_PAT<F32FMA_ftzV4, doFMAF32AGG_ftz>;
603defm V4FMADF32ext_ftz : V4FPCONTRACT32_SUB_PAT<F32MAD_ftzV4, doFMADF32_ftz>;
604defm V4FMAF32ext  : V4FPCONTRACT32_SUB_PAT<F32FMAV4, doFMAF32AGG>;
605defm V4FMADF32ext : V4FPCONTRACT32_SUB_PAT<F32MADV4, doFMADF32>;
606
607multiclass V2FPCONTRACT64_SUB_PAT<NVPTXInst Inst, Predicate Pred> {
608  def : Pat<(fsub V2F64Regs:$a, (fmul V2F64Regs:$b, V2F64Regs:$c)),
609          (Inst (VNegv2f64 V2F64Regs:$b), V2F64Regs:$c, V2F64Regs:$a)>,
610          Requires<[Pred]>;
611
612  def : Pat<(fsub (fmul V2F64Regs:$a, V2F64Regs:$b), V2F64Regs:$c),
613          (Inst V2F64Regs:$a, V2F64Regs:$b, (VNegv2f64 V2F64Regs:$c))>,
614          Requires<[Pred]>;
615}
616
617defm V2FMAF64ext : V2FPCONTRACT64_SUB_PAT<F64FMAV2, doFMAF64AGG>;
618
619class VecModStr<string vecsize, string elem, string extra, string l="">
620{
621  string t1 = !strconcat("${c", elem);
622  string t2 = !strconcat(t1, ":vecv");
623  string t3 = !strconcat(t2, vecsize);
624  string t4 = !strconcat(t3, extra);
625  string t5 = !strconcat(t4, l);
626  string s =  !strconcat(t5, "}");
627}
628class ShuffleOneLine<string vecsize, string elem, string type>
629{
630  string t1 = VecModStr<vecsize, elem, "comm", "1">.s;
631  string t2 = !strconcat(t1, "mov.");
632  string t3 = !strconcat(t2, type);
633  string t4 = !strconcat(t3, " \t${dst}_");
634  string t5 = !strconcat(t4, elem);
635  string t6 = !strconcat(t5, ", $src1");
636  string t7 = !strconcat(t6, VecModStr<vecsize, elem, "pos">.s);
637  string t8 = !strconcat(t7, ";\n\t");
638  string t9 = !strconcat(t8, VecModStr<vecsize, elem, "comm", "2">.s);
639  string t10 = !strconcat(t9, "mov.");
640  string t11 = !strconcat(t10, type);
641  string t12 = !strconcat(t11, " \t${dst}_");
642  string t13 = !strconcat(t12, elem);
643  string t14 = !strconcat(t13, ", $src2");
644  string t15 = !strconcat(t14, VecModStr<vecsize, elem, "pos">.s);
645  string s =   !strconcat(t15, ";");
646}
647class ShuffleAsmStr2<string type>
648{
649  string t1 = ShuffleOneLine<"2", "0", type>.s;
650  string t2 = !strconcat(t1, "\n\t");
651  string s  = !strconcat(t2, ShuffleOneLine<"2", "1", type>.s);
652}
653class ShuffleAsmStr4<string type>
654{
655  string t1 = ShuffleOneLine<"4", "0", type>.s;
656  string t2 = !strconcat(t1, "\n\t");
657  string t3 = !strconcat(t2, ShuffleOneLine<"4", "1", type>.s);
658  string t4 = !strconcat(t3, "\n\t");
659  string t5 = !strconcat(t4, ShuffleOneLine<"4", "2", type>.s);
660  string t6 = !strconcat(t5, "\n\t");
661  string s  = !strconcat(t6, ShuffleOneLine<"4", "3", type>.s);
662}
663
664let neverHasSideEffects=1, VecInstType=isVecShuffle.Value in {
665def VecShuffle_v4f32 : NVPTXVecInst<(outs V4F32Regs:$dst),
666                       (ins  V4F32Regs:$src1, V4F32Regs:$src2,
667                             i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
668                 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
669                                 ShuffleAsmStr4<"f32">.s),
670                       [], FMOV32rr>;
671
672def VecShuffle_v4i32 : NVPTXVecInst<(outs V4I32Regs:$dst),
673                       (ins  V4I32Regs:$src1, V4I32Regs:$src2,
674                             i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
675                 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
676                                 ShuffleAsmStr4<"u32">.s),
677                       [], IMOV32rr>;
678
679def VecShuffle_v4i16 : NVPTXVecInst<(outs V4I16Regs:$dst),
680                       (ins  V4I16Regs:$src1, V4I16Regs:$src2,
681                             i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
682                 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
683                                 ShuffleAsmStr4<"u16">.s),
684                       [], IMOV16rr>;
685
686def VecShuffle_v4i8 : NVPTXVecInst<(outs V4I8Regs:$dst),
687                       (ins  V4I8Regs:$src1, V4I8Regs:$src2,
688                             i8imm:$c0, i8imm:$c1, i8imm:$c2, i8imm:$c3),
689                 !strconcat("//Mov $dst, $src1, $src2, $c0, $c1, $c2, $c3;\n\t",
690                                 ShuffleAsmStr4<"u16">.s),
691                       [], IMOV8rr>;
692
693def VecShuffle_v2f32 : NVPTXVecInst<(outs V2F32Regs:$dst),
694                       (ins  V2F32Regs:$src1, V2F32Regs:$src2,
695                             i8imm:$c0, i8imm:$c1),
696                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
697                                 ShuffleAsmStr2<"f32">.s),
698                       [], FMOV32rr>;
699
700def VecShuffle_v2i32 : NVPTXVecInst<(outs V2I32Regs:$dst),
701                       (ins  V2I32Regs:$src1, V2I32Regs:$src2,
702                             i8imm:$c0, i8imm:$c1),
703                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
704                                 ShuffleAsmStr2<"u32">.s),
705                       [], IMOV32rr>;
706
707def VecShuffle_v2i8 : NVPTXVecInst<(outs V2I8Regs:$dst),
708                       (ins  V2I8Regs:$src1, V2I8Regs:$src2,
709                             i8imm:$c0, i8imm:$c1),
710                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
711                                 ShuffleAsmStr2<"u16">.s),
712                       [], IMOV8rr>;
713
714def VecShuffle_v2i16 : NVPTXVecInst<(outs V2I16Regs:$dst),
715                       (ins  V2I16Regs:$src1, V2I16Regs:$src2,
716                             i8imm:$c0, i8imm:$c1),
717                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
718                                 ShuffleAsmStr2<"u16">.s),
719                       [], IMOV16rr>;
720
721def VecShuffle_v2f64 : NVPTXVecInst<(outs V2F64Regs:$dst),
722                       (ins  V2F64Regs:$src1, V2F64Regs:$src2,
723                             i8imm:$c0, i8imm:$c1),
724                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
725                                 ShuffleAsmStr2<"f64">.s),
726                       [], FMOV64rr>;
727
728def VecShuffle_v2i64 : NVPTXVecInst<(outs V2I64Regs:$dst),
729                       (ins  V2I64Regs:$src1, V2I64Regs:$src2,
730                             i8imm:$c0, i8imm:$c1),
731                       !strconcat("//Mov $dst, $src1, $src2, $c0, $c1;\n\t",
732                                 ShuffleAsmStr2<"u64">.s),
733                       [], IMOV64rr>;
734}
735
736def ShuffleMask0 : SDNodeXForm<vector_shuffle, [{
737  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
738  return CurDAG->getTargetConstant(SVOp->getMaskElt(0), MVT::i32);
739}]>;
740def ShuffleMask1 : SDNodeXForm<vector_shuffle, [{
741  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
742  return CurDAG->getTargetConstant(SVOp->getMaskElt(1), MVT::i32);
743}]>;
744def ShuffleMask2 : SDNodeXForm<vector_shuffle, [{
745  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
746  return CurDAG->getTargetConstant(SVOp->getMaskElt(2), MVT::i32);
747}]>;
748def ShuffleMask3 : SDNodeXForm<vector_shuffle, [{
749  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
750  return CurDAG->getTargetConstant(SVOp->getMaskElt(3), MVT::i32);
751}]>;
752
753// The spurious call is here to silence a compiler warning about N being
754// unused.
755def vec_shuf : PatFrag<(ops node:$lhs, node:$rhs),
756                       (vector_shuffle node:$lhs, node:$rhs),
757                       [{ N->getGluedNode(); return true; }]>;
758
759def : Pat<(v2f64 (vec_shuf:$op V2F64Regs:$src1, V2F64Regs:$src2)),
760          (VecShuffle_v2f64 V2F64Regs:$src1, V2F64Regs:$src2,
761                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
762
763def : Pat<(v4f32 (vec_shuf:$op V4F32Regs:$src1, V4F32Regs:$src2)),
764          (VecShuffle_v4f32 V4F32Regs:$src1, V4F32Regs:$src2,
765                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
766                            (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
767
768def : Pat<(v2f32 (vec_shuf:$op V2F32Regs:$src1, V2F32Regs:$src2)),
769          (VecShuffle_v2f32 V2F32Regs:$src1, V2F32Regs:$src2,
770                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
771
772def : Pat<(v2i64 (vec_shuf:$op V2I64Regs:$src1, V2I64Regs:$src2)),
773          (VecShuffle_v2i64 V2I64Regs:$src1, V2I64Regs:$src2,
774                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
775
776def : Pat<(v4i32 (vec_shuf:$op V4I32Regs:$src1, V4I32Regs:$src2)),
777          (VecShuffle_v4i32 V4I32Regs:$src1, V4I32Regs:$src2,
778                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
779                            (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
780
781def : Pat<(v2i32 (vec_shuf:$op V2I32Regs:$src1, V2I32Regs:$src2)),
782          (VecShuffle_v2i32 V2I32Regs:$src1, V2I32Regs:$src2,
783                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
784
785def : Pat<(v4i16 (vec_shuf:$op V4I16Regs:$src1, V4I16Regs:$src2)),
786          (VecShuffle_v4i16 V4I16Regs:$src1, V4I16Regs:$src2,
787                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
788                            (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
789
790def : Pat<(v2i16 (vec_shuf:$op V2I16Regs:$src1, V2I16Regs:$src2)),
791          (VecShuffle_v2i16 V2I16Regs:$src1, V2I16Regs:$src2,
792                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
793
794def : Pat<(v4i8 (vec_shuf:$op V4I8Regs:$src1, V4I8Regs:$src2)),
795          (VecShuffle_v4i8 V4I8Regs:$src1, V4I8Regs:$src2,
796                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op),
797                            (ShuffleMask2 node:$op), (ShuffleMask3 node:$op))>;
798
799def : Pat<(v2i8 (vec_shuf:$op V2I8Regs:$src1, V2I8Regs:$src2)),
800          (VecShuffle_v2i8 V2I8Regs:$src1, V2I8Regs:$src2,
801                            (ShuffleMask0 node:$op), (ShuffleMask1 node:$op))>;
802
803class Build_Vector2<string asmstr, NVPTXRegClass vclass, NVPTXRegClass sclass,
804  NVPTXInst si>
805                   : NVPTXVecInst<(outs vclass:$dst),
806                   (ins  sclass:$a1, sclass:$a2),
807                   !strconcat(asmstr, "\t${dst:vecfull}, {{$a1, $a2}};"),
808                   [(set vclass:$dst, (build_vector sclass:$a1, sclass:$a2))],
809                   si>;
810class Build_Vector4<string asmstr, NVPTXRegClass vclass, NVPTXRegClass sclass,
811  NVPTXInst si>
812                   : NVPTXVecInst<(outs vclass:$dst),
813                   (ins  sclass:$a1, sclass:$a2, sclass:$a3, sclass:$a4),
814               !strconcat(asmstr, "\t${dst:vecfull}, {{$a1, $a2, $a3, $a4}};"),
815                   [(set vclass:$dst,
816                     (build_vector sclass:$a1, sclass:$a2,
817                       sclass:$a3, sclass:$a4))], si>;
818
819let isAsCheapAsAMove=1, VecInstType=isVecBuild.Value in {
820def Build_Vector2_f32 : Build_Vector2<"mov.v2.f32", V2F32Regs, Float32Regs,
821  FMOV32rr>;
822def Build_Vector2_f64 : Build_Vector2<"mov.v2.f64", V2F64Regs, Float64Regs,
823  FMOV64rr>;
824
825def Build_Vector2_i32 : Build_Vector2<"mov.v2.u32", V2I32Regs, Int32Regs,
826  IMOV32rr>;
827def Build_Vector2_i64 : Build_Vector2<"mov.v2.u64", V2I64Regs, Int64Regs,
828  IMOV64rr>;
829def Build_Vector2_i16 : Build_Vector2<"mov.v2.u16", V2I16Regs, Int16Regs,
830  IMOV16rr>;
831def Build_Vector2_i8  : Build_Vector2<"mov.v2.u16",  V2I8Regs,  Int8Regs,
832  IMOV8rr>;
833
834def Build_Vector4_f32 : Build_Vector4<"mov.v4.f32", V4F32Regs, Float32Regs,
835  FMOV32rr>;
836
837def Build_Vector4_i32 : Build_Vector4<"mov.v4.u32", V4I32Regs, Int32Regs,
838  IMOV32rr>;
839def Build_Vector4_i16 : Build_Vector4<"mov.v4.u16", V4I16Regs, Int16Regs,
840  IMOV16rr>;
841def Build_Vector4_i8  : Build_Vector4<"mov.v4.u16", V4I8Regs, Int8Regs,
842  IMOV8rr>;
843}
844
845class Vec_Move<string asmstr, NVPTXRegClass vclass, NVPTXInst sop=NOP>
846                 : NVPTXVecInst<(outs vclass:$dst), (ins vclass:$src),
847                   !strconcat(asmstr, "\t${dst:vecfull}, ${src:vecfull};"),
848                   [], sop>;
849
850let isAsCheapAsAMove=1, neverHasSideEffects=1, IsSimpleMove=1,
851  VecInstType=isVecOther.Value in {
852def V4f32Mov : Vec_Move<"mov.v4.f32", V4F32Regs, FMOV32rr>;
853def V2f32Mov : Vec_Move<"mov.v2.f32", V2F32Regs, FMOV32rr>;
854
855def V4i32Mov : Vec_Move<"mov.v4.u32", V4I32Regs, IMOV32rr>;
856def V2i32Mov : Vec_Move<"mov.v2.u32", V2I32Regs, IMOV32rr>;
857
858def V4i16Mov : Vec_Move<"mov.v4.u16", V4I16Regs, IMOV16rr>;
859def V2i16Mov : Vec_Move<"mov.v2.u16", V2I16Regs, IMOV16rr>;
860
861def V4i8Mov : Vec_Move<"mov.v4.u16", V4I8Regs, IMOV8rr>;
862def V2i8Mov : Vec_Move<"mov.v2.u16", V2I8Regs, IMOV8rr>;
863
864def V2f64Mov : Vec_Move<"mov.v2.f64", V2F64Regs, FMOV64rr>;
865def V2i64Mov : Vec_Move<"mov.v2.u64", V2I64Regs, IMOV64rr>;
866}
867
868// extract subvector patterns
869def extract_subvec : SDNode<"ISD::EXTRACT_SUBVECTOR",
870                        SDTypeProfile<1, 2, [SDTCisPtrTy<2>]>>;
871
872def : Pat<(v2f32 (extract_subvec V4F32Regs:$src, 0)),
873                 (Build_Vector2_f32 (V4f32Extract V4F32Regs:$src, 0),
874                                    (V4f32Extract V4F32Regs:$src, 1))>;
875def : Pat<(v2f32 (extract_subvec V4F32Regs:$src, 2)),
876                 (Build_Vector2_f32 (V4f32Extract V4F32Regs:$src, 2),
877                                    (V4f32Extract V4F32Regs:$src, 3))>;
878def : Pat<(v2i32 (extract_subvec V4I32Regs:$src, 0)),
879                 (Build_Vector2_i32 (V4i32Extract V4I32Regs:$src, 0),
880                                    (V4i32Extract V4I32Regs:$src, 1))>;
881def : Pat<(v2i32 (extract_subvec V4I32Regs:$src, 2)),
882                 (Build_Vector2_i32 (V4i32Extract V4I32Regs:$src, 2),
883                                    (V4i32Extract V4I32Regs:$src, 3))>;
884def : Pat<(v2i16 (extract_subvec V4I16Regs:$src, 0)),
885                 (Build_Vector2_i16 (V4i16Extract V4I16Regs:$src, 0),
886                                    (V4i16Extract V4I16Regs:$src, 1))>;
887def : Pat<(v2i16 (extract_subvec V4I16Regs:$src, 2)),
888                 (Build_Vector2_i16 (V4i16Extract V4I16Regs:$src, 2),
889                                    (V4i16Extract V4I16Regs:$src, 3))>;
890def : Pat<(v2i8 (extract_subvec V4I8Regs:$src, 0)),
891                 (Build_Vector2_i8 (V4i8Extract V4I8Regs:$src, 0),
892                                    (V4i8Extract V4I8Regs:$src, 1))>;
893def : Pat<(v2i8 (extract_subvec V4I8Regs:$src, 2)),
894                 (Build_Vector2_i8 (V4i8Extract V4I8Regs:$src, 2),
895                                    (V4i8Extract V4I8Regs:$src, 3))>;
896
897// Select instructions
898class Select_OneLine<string type, string pos> {
899  string t1 = !strconcat("selp.", type);
900  string t2 = !strconcat(t1, " \t${dst}_");
901  string t3 = !strconcat(t2, pos);
902  string t4 = !strconcat(t3, ", ${src1}_");
903  string t5 = !strconcat(t4, pos);
904  string t6 = !strconcat(t5, ", ${src2}_");
905  string t7 = !strconcat(t6, pos);
906  string s  = !strconcat(t7, ", $p;");
907}
908
909class Select_Str2<string type> {
910  string t1 = Select_OneLine<type, "0">.s;
911  string t2 = !strconcat(t1, "\n\t");
912  string s  = !strconcat(t2, Select_OneLine<type, "1">.s);
913}
914
915class Select_Str4<string type> {
916  string t1 = Select_OneLine<type, "0">.s;
917  string t2 = !strconcat(t1, "\n\t");
918  string t3 = !strconcat(t2, Select_OneLine<type, "1">.s);
919  string t4 = !strconcat(t3, "\n\t");
920  string t5 = !strconcat(t4, Select_OneLine<type, "2">.s);
921  string t6 = !strconcat(t5, "\n\t");
922  string s  = !strconcat(t6, Select_OneLine<type, "3">.s);
923
924}
925
926class Vec_Select<NVPTXRegClass vclass, string asmstr, NVPTXInst sop>
927      : NVPTXVecInst<(outs vclass:$dst),
928                     (ins  vclass:$src1, vclass:$src2, Int1Regs:$p),
929                     asmstr,
930                     [(set vclass:$dst, (select Int1Regs:$p, vclass:$src1,
931                       vclass:$src2))],
932                     sop>;
933
934let VecInstType=isVecOther.Value in {
935def V2I64_Select : Vec_Select<V2I64Regs, Select_Str2<"b64">.s, SELECTi64rr>;
936def V4I32_Select : Vec_Select<V4I32Regs, Select_Str4<"b32">.s, SELECTi32rr>;
937def V2I32_Select : Vec_Select<V2I32Regs, Select_Str2<"b32">.s, SELECTi32rr>;
938def V4I16_Select : Vec_Select<V4I16Regs, Select_Str4<"b16">.s, SELECTi16rr>;
939def V2I16_Select : Vec_Select<V2I16Regs, Select_Str2<"b16">.s, SELECTi16rr>;
940def V4I8_Select  : Vec_Select<V4I8Regs,  Select_Str4<"b16">.s, SELECTi8rr>;
941def V2I8_Select  : Vec_Select<V2I8Regs,  Select_Str2<"b16">.s, SELECTi8rr>;
942
943def V2F64_Select : Vec_Select<V2F64Regs, Select_Str2<"f64">.s, SELECTf64rr>;
944def V4F32_Select : Vec_Select<V4F32Regs, Select_Str4<"f32">.s, SELECTf32rr>;
945def V2F32_Select : Vec_Select<V2F32Regs, Select_Str2<"f32">.s, SELECTf32rr>;
946}
947
948// Comparison instructions
949
950// setcc convenience fragments.
951def vsetoeq : PatFrag<(ops node:$lhs, node:$rhs),
952                      (setcc node:$lhs, node:$rhs, SETOEQ)>;
953def vsetogt : PatFrag<(ops node:$lhs, node:$rhs),
954                      (setcc node:$lhs, node:$rhs, SETOGT)>;
955def vsetoge : PatFrag<(ops node:$lhs, node:$rhs),
956                      (setcc node:$lhs, node:$rhs, SETOGE)>;
957def vsetolt : PatFrag<(ops node:$lhs, node:$rhs),
958                      (setcc node:$lhs, node:$rhs, SETOLT)>;
959def vsetole : PatFrag<(ops node:$lhs, node:$rhs),
960                      (setcc node:$lhs, node:$rhs, SETOLE)>;
961def vsetone : PatFrag<(ops node:$lhs, node:$rhs),
962                      (setcc node:$lhs, node:$rhs, SETONE)>;
963def vseto   : PatFrag<(ops node:$lhs, node:$rhs),
964                      (setcc node:$lhs, node:$rhs, SETO)>;
965def vsetuo  : PatFrag<(ops node:$lhs, node:$rhs),
966                      (setcc node:$lhs, node:$rhs, SETUO)>;
967def vsetueq : PatFrag<(ops node:$lhs, node:$rhs),
968                      (setcc node:$lhs, node:$rhs, SETUEQ)>;
969def vsetugt : PatFrag<(ops node:$lhs, node:$rhs),
970                      (setcc node:$lhs, node:$rhs, SETUGT)>;
971def vsetuge : PatFrag<(ops node:$lhs, node:$rhs),
972                      (setcc node:$lhs, node:$rhs, SETUGE)>;
973def vsetult : PatFrag<(ops node:$lhs, node:$rhs),
974                      (setcc node:$lhs, node:$rhs, SETULT)>;
975def vsetule : PatFrag<(ops node:$lhs, node:$rhs),
976                      (setcc node:$lhs, node:$rhs, SETULE)>;
977def vsetune : PatFrag<(ops node:$lhs, node:$rhs),
978                      (setcc node:$lhs, node:$rhs, SETUNE)>;
979def vseteq  : PatFrag<(ops node:$lhs, node:$rhs),
980                      (setcc node:$lhs, node:$rhs, SETEQ)>;
981def vsetgt  : PatFrag<(ops node:$lhs, node:$rhs),
982                      (setcc node:$lhs, node:$rhs, SETGT)>;
983def vsetge  : PatFrag<(ops node:$lhs, node:$rhs),
984                      (setcc node:$lhs, node:$rhs, SETGE)>;
985def vsetlt  : PatFrag<(ops node:$lhs, node:$rhs),
986                      (setcc node:$lhs, node:$rhs, SETLT)>;
987def vsetle  : PatFrag<(ops node:$lhs, node:$rhs),
988                      (setcc node:$lhs, node:$rhs, SETLE)>;
989def vsetne  : PatFrag<(ops node:$lhs, node:$rhs),
990                      (setcc node:$lhs, node:$rhs, SETNE)>;
991
992class Vec_Compare<PatFrag op, NVPTXRegClass outrclass, NVPTXRegClass inrclass,
993  NVPTXInst sop>
994    : NVPTXVecInst<(outs outrclass:$dst),
995                   (ins  inrclass:$a, inrclass:$b),
996                   "Unsupported",
997                   [(set outrclass:$dst, (op inrclass:$a, inrclass:$b))],
998                   sop>;
999
1000multiclass Vec_Compare_All<PatFrag op,
1001                           NVPTXInst inst8,
1002                           NVPTXInst inst16,
1003                           NVPTXInst inst32,
1004                           NVPTXInst inst64>
1005{
1006  def  V2I8 : Vec_Compare<op, V2I8Regs,  V2I8Regs,  inst8>;
1007  def  V4I8 : Vec_Compare<op, V4I8Regs,  V4I8Regs,  inst8>;
1008  def V2I16 : Vec_Compare<op, V2I16Regs, V2I16Regs, inst16>;
1009  def V4I16 : Vec_Compare<op, V4I16Regs, V4I16Regs, inst16>;
1010  def V2I32 : Vec_Compare<op, V2I32Regs, V2I32Regs, inst32>;
1011  def V4I32 : Vec_Compare<op, V4I32Regs, V4I32Regs, inst32>;
1012  def V2I64 : Vec_Compare<op, V2I64Regs, V2I64Regs, inst64>;
1013}
1014
1015let VecInstType=isVecOther.Value in {
1016  defm VecSGT : Vec_Compare_All<vsetgt,  ISetSGTi8rr_toi8, ISetSGTi16rr_toi16,
1017    ISetSGTi32rr_toi32, ISetSGTi64rr_toi64>;
1018  defm VecUGT : Vec_Compare_All<vsetugt, ISetUGTi8rr_toi8, ISetUGTi16rr_toi16,
1019    ISetUGTi32rr_toi32, ISetUGTi64rr_toi64>;
1020  defm VecSLT : Vec_Compare_All<vsetlt,  ISetSLTi8rr_toi8, ISetSLTi16rr_toi16,
1021    ISetSLTi32rr_toi32, ISetSLTi64rr_toi64>;
1022  defm VecULT : Vec_Compare_All<vsetult, ISetULTi8rr_toi8, ISetULTi16rr_toi16,
1023    ISetULTi32rr_toi32, ISetULTi64rr_toi64>;
1024  defm VecSGE : Vec_Compare_All<vsetge,  ISetSGEi8rr_toi8, ISetSGEi16rr_toi16,
1025    ISetSGEi32rr_toi32, ISetSGEi64rr_toi64>;
1026  defm VecUGE : Vec_Compare_All<vsetuge, ISetUGEi8rr_toi8, ISetUGEi16rr_toi16,
1027    ISetUGEi32rr_toi32, ISetUGEi64rr_toi64>;
1028  defm VecSLE : Vec_Compare_All<vsetle,  ISetSLEi8rr_toi8, ISetSLEi16rr_toi16,
1029    ISetSLEi32rr_toi32, ISetSLEi64rr_toi64>;
1030  defm VecULE : Vec_Compare_All<vsetule, ISetULEi8rr_toi8, ISetULEi16rr_toi16,
1031    ISetULEi32rr_toi32, ISetULEi64rr_toi64>;
1032  defm VecSEQ : Vec_Compare_All<vseteq,  ISetSEQi8rr_toi8, ISetSEQi16rr_toi16,
1033    ISetSEQi32rr_toi32, ISetSEQi64rr_toi64>;
1034  defm VecUEQ : Vec_Compare_All<vsetueq, ISetUEQi8rr_toi8, ISetUEQi16rr_toi16,
1035    ISetUEQi32rr_toi32, ISetUEQi64rr_toi64>;
1036  defm VecSNE : Vec_Compare_All<vsetne,  ISetSNEi8rr_toi8, ISetSNEi16rr_toi16,
1037    ISetSNEi32rr_toi32, ISetSNEi64rr_toi64>;
1038  defm VecUNE : Vec_Compare_All<vsetune, ISetUNEi8rr_toi8, ISetUNEi16rr_toi16,
1039    ISetUNEi32rr_toi32, ISetUNEi64rr_toi64>;
1040}
1041
1042multiclass FVec_Compare_All<PatFrag op,
1043                            NVPTXInst instf32,
1044                            NVPTXInst instf64>
1045{
1046  def V2F32 : Vec_Compare<op, V2I32Regs, V2F32Regs, instf32>;
1047  def V4F32 : Vec_Compare<op, V4I32Regs, V4F32Regs, instf32>;
1048  def V2F64 : Vec_Compare<op, V2I64Regs, V2F64Regs, instf64>;
1049}
1050
1051let VecInstType=isVecOther.Value in {
1052  defm FVecGT :  FVec_Compare_All<vsetogt, FSetGTf32rr_toi32,
1053    FSetGTf64rr_toi64>;
1054  defm FVecLT :  FVec_Compare_All<vsetolt, FSetLTf32rr_toi32,
1055    FSetLTf64rr_toi64>;
1056  defm FVecGE :  FVec_Compare_All<vsetoge, FSetGEf32rr_toi32,
1057    FSetGEf64rr_toi64>;
1058  defm FVecLE :  FVec_Compare_All<vsetole, FSetLEf32rr_toi32,
1059    FSetLEf64rr_toi64>;
1060  defm FVecEQ :  FVec_Compare_All<vsetoeq, FSetEQf32rr_toi32,
1061    FSetEQf64rr_toi64>;
1062  defm FVecNE :  FVec_Compare_All<vsetone, FSetNEf32rr_toi32,
1063    FSetNEf64rr_toi64>;
1064
1065  defm FVecUGT :  FVec_Compare_All<vsetugt, FSetUGTf32rr_toi32,
1066    FSetUGTf64rr_toi64>;
1067  defm FVecULT :  FVec_Compare_All<vsetult, FSetULTf32rr_toi32,
1068    FSetULTf64rr_toi64>;
1069  defm FVecUGE :  FVec_Compare_All<vsetuge, FSetUGEf32rr_toi32,
1070    FSetUGEf64rr_toi64>;
1071  defm FVecULE :  FVec_Compare_All<vsetule, FSetULEf32rr_toi32,
1072    FSetULEf64rr_toi64>;
1073  defm FVecUEQ :  FVec_Compare_All<vsetueq, FSetUEQf32rr_toi32,
1074    FSetUEQf64rr_toi64>;
1075  defm FVecUNE :  FVec_Compare_All<vsetune, FSetUNEf32rr_toi32,
1076    FSetUNEf64rr_toi64>;
1077
1078  defm FVecNUM :  FVec_Compare_All<vseto,  FSetNUMf32rr_toi32,
1079    FSetNUMf64rr_toi64>;
1080  defm FVecNAN :  FVec_Compare_All<vsetuo, FSetNANf32rr_toi32,
1081    FSetNANf64rr_toi64>;
1082}
1083
1084class LoadParamScalar4Inst<NVPTXRegClass regclass, string opstr> :
1085      NVPTXInst<(outs regclass:$d1, regclass:$d2, regclass:$d3, regclass:$d4),
1086                (ins i32imm:$a, i32imm:$b),
1087                !strconcat(!strconcat("ld.param", opstr),
1088                  "\t{{$d1, $d2, $d3, $d4}}, [retval0+$b];"), []>;
1089
1090class LoadParamScalar2Inst<NVPTXRegClass regclass, string opstr> :
1091      NVPTXInst<(outs regclass:$d1, regclass:$d2),
1092                (ins i32imm:$a, i32imm:$b),
1093                !strconcat(!strconcat("ld.param", opstr),
1094                  "\t{{$d1, $d2}}, [retval0+$b];"), []>;
1095
1096
1097class StoreParamScalar4Inst<NVPTXRegClass regclass, string opstr> :
1098      NVPTXInst<(outs),
1099                (ins regclass:$s1, regclass:$s2, regclass:$s3, regclass:$s4,
1100                  i32imm:$a, i32imm:$b),
1101                !strconcat(!strconcat("st.param", opstr),
1102                  "\t[param$a+$b], {{$s1, $s2, $s3, $s4}};"), []>;
1103
1104class StoreParamScalar2Inst<NVPTXRegClass regclass, string opstr> :
1105      NVPTXInst<(outs),
1106                (ins regclass:$s1, regclass:$s2, i32imm:$a, i32imm:$b),
1107                !strconcat(!strconcat("st.param", opstr),
1108                  "\t[param$a+$b], {{$s1, $s2}};"), []>;
1109
1110class StoreRetvalScalar4Inst<NVPTXRegClass regclass, string opstr> :
1111      NVPTXInst<(outs),
1112                (ins regclass:$s1, regclass:$s2, regclass:$s3, regclass:$s4,
1113                  i32imm:$a),
1114                !strconcat(!strconcat("st.param", opstr),
1115                  "\t[func_retval+$a], {{$s1, $s2, $s3, $s4}};"), []>;
1116
1117class StoreRetvalScalar2Inst<NVPTXRegClass regclass, string opstr> :
1118      NVPTXInst<(outs),
1119                (ins regclass:$s1, regclass:$s2, i32imm:$a),
1120                !strconcat(!strconcat("st.param", opstr),
1121                  "\t[func_retval+$a], {{$s1, $s2}};"), []>;
1122
1123def LoadParamScalar4I32 : LoadParamScalar4Inst<Int32Regs, ".v4.b32">;
1124def LoadParamScalar4I16 : LoadParamScalar4Inst<Int16Regs, ".v4.b16">;
1125def LoadParamScalar4I8  : LoadParamScalar4Inst<Int8Regs, ".v4.b8">;
1126
1127def LoadParamScalar2I64 : LoadParamScalar2Inst<Int32Regs, ".v2.b64">;
1128def LoadParamScalar2I32 : LoadParamScalar2Inst<Int32Regs, ".v2.b32">;
1129def LoadParamScalar2I16 : LoadParamScalar2Inst<Int32Regs, ".v2.b16">;
1130def LoadParamScalar2I8  : LoadParamScalar2Inst<Int32Regs, ".v2.b8">;
1131
1132def LoadParamScalar4F32 : LoadParamScalar4Inst<Float32Regs, ".v4.f32">;
1133def LoadParamScalar2F32 : LoadParamScalar2Inst<Float32Regs, ".v2.f32">;
1134def LoadParamScalar2F64 : LoadParamScalar2Inst<Float64Regs, ".v2.f64">;
1135
1136def StoreParamScalar4I32 : StoreParamScalar4Inst<Int32Regs, ".v4.b32">;
1137def StoreParamScalar4I16 : StoreParamScalar4Inst<Int16Regs, ".v4.b16">;
1138def StoreParamScalar4I8  : StoreParamScalar4Inst<Int8Regs, ".v4.b8">;
1139
1140def StoreParamScalar2I64 : StoreParamScalar2Inst<Int64Regs, ".v2.b64">;
1141def StoreParamScalar2I32 : StoreParamScalar2Inst<Int32Regs, ".v2.b32">;
1142def StoreParamScalar2I16 : StoreParamScalar2Inst<Int16Regs, ".v2.b16">;
1143def StoreParamScalar2I8  : StoreParamScalar2Inst<Int8Regs, ".v2.b8">;
1144
1145def StoreParamScalar4F32 : StoreParamScalar4Inst<Float32Regs, ".v4.f32">;
1146def StoreParamScalar2F32 : StoreParamScalar2Inst<Float32Regs, ".v2.f32">;
1147def StoreParamScalar2F64 : StoreParamScalar2Inst<Float64Regs, ".v2.f64">;
1148
1149def StoreRetvalScalar4I32 : StoreRetvalScalar4Inst<Int32Regs, ".v4.b32">;
1150def StoreRetvalScalar4I16 : StoreRetvalScalar4Inst<Int16Regs, ".v4.b16">;
1151def StoreRetvalScalar4I8  : StoreRetvalScalar4Inst<Int8Regs, ".v4.b8">;
1152
1153def StoreRetvalScalar2I64 : StoreRetvalScalar2Inst<Int64Regs, ".v2.b64">;
1154def StoreRetvalScalar2I32 : StoreRetvalScalar2Inst<Int32Regs, ".v2.b32">;
1155def StoreRetvalScalar2I16 : StoreRetvalScalar2Inst<Int16Regs, ".v2.b16">;
1156def StoreRetvalScalar2I8  : StoreRetvalScalar2Inst<Int8Regs, ".v2.b8">;
1157
1158def StoreRetvalScalar4F32 : StoreRetvalScalar4Inst<Float32Regs, ".v4.f32">;
1159def StoreRetvalScalar2F32 : StoreRetvalScalar2Inst<Float32Regs, ".v2.f32">;
1160def StoreRetvalScalar2F64 : StoreRetvalScalar2Inst<Float64Regs, ".v2.f64">;
1161
1162class LoadParamVecInst<NVPTXRegClass regclass, string opstr, NVPTXInst sop=NOP>:
1163      NVPTXVecInst<(outs regclass:$dst), (ins i32imm:$a, i32imm:$b),
1164                "loadparam : $dst <- [$a, $b]",
1165                [(set regclass:$dst, (LoadParam (i32 imm:$a), (i32 imm:$b)))],
1166                sop>;
1167
1168class StoreParamVecInst<NVPTXRegClass regclass, string opstr, NVPTXInst sop=NOP>
1169      : NVPTXVecInst<(outs), (ins regclass:$val, i32imm:$a, i32imm:$b),
1170                "storeparam : [$a, $b] <- $val",
1171                [(StoreParam (i32 imm:$a), (i32 imm:$b), regclass:$val)], sop>;
1172
1173class StoreRetvalVecInst<NVPTXRegClass regclass, string opstr,
1174  NVPTXInst sop=NOP>
1175      : NVPTXVecInst<(outs), (ins regclass:$val, i32imm:$a),
1176                "storeretval : retval[$a] <- $val",
1177                [(StoreRetval (i32 imm:$a), regclass:$val)], sop>;
1178
1179let VecInstType=isVecLD.Value in {
1180def LoadParamV4I32  : LoadParamVecInst<V4I32Regs, ".v4.b32",
1181  LoadParamScalar4I32>;
1182def LoadParamV4I16  : LoadParamVecInst<V4I16Regs, ".v4.b16",
1183  LoadParamScalar4I16>;
1184def LoadParamV4I8   : LoadParamVecInst<V4I8Regs, ".v4.b8",
1185  LoadParamScalar4I8>;
1186
1187def LoadParamV2I64  : LoadParamVecInst<V2I64Regs, ".v2.b64",
1188  LoadParamScalar2I64>;
1189def LoadParamV2I32  : LoadParamVecInst<V2I32Regs, ".v2.b32",
1190  LoadParamScalar2I32>;
1191def LoadParamV2I16  : LoadParamVecInst<V2I16Regs, ".v2.b16",
1192  LoadParamScalar2I16>;
1193def LoadParamV2I8   : LoadParamVecInst<V2I8Regs, ".v2.b8",
1194  LoadParamScalar2I8>;
1195
1196def LoadParamV4F32  : LoadParamVecInst<V4F32Regs, ".v4.f32",
1197  LoadParamScalar4F32>;
1198def LoadParamV2F32  : LoadParamVecInst<V2F32Regs, ".v2.f32",
1199  LoadParamScalar2F32>;
1200def LoadParamV2F64  : LoadParamVecInst<V2F64Regs, ".v2.f64",
1201  LoadParamScalar2F64>;
1202}
1203
1204let VecInstType=isVecST.Value in {
1205def StoreParamV4I32  : StoreParamVecInst<V4I32Regs, ".v4.b32",
1206  StoreParamScalar4I32>;
1207def StoreParamV4I16  : StoreParamVecInst<V4I16Regs, ".v4.b16",
1208  StoreParamScalar4I16>;
1209def StoreParamV4I8   : StoreParamVecInst<V4I8Regs, ".v4.b8",
1210  StoreParamScalar4I8>;
1211
1212def StoreParamV2I64  : StoreParamVecInst<V2I64Regs, ".v2.b64",
1213  StoreParamScalar2I64>;
1214def StoreParamV2I32  : StoreParamVecInst<V2I32Regs, ".v2.b32",
1215  StoreParamScalar2I32>;
1216def StoreParamV2I16  : StoreParamVecInst<V2I16Regs, ".v2.b16",
1217  StoreParamScalar2I16>;
1218def StoreParamV2I8   : StoreParamVecInst<V2I8Regs, ".v2.b8",
1219  StoreParamScalar2I8>;
1220
1221def StoreParamV4F32  : StoreParamVecInst<V4F32Regs, ".v4.f32",
1222  StoreParamScalar4F32>;
1223def StoreParamV2F32  : StoreParamVecInst<V2F32Regs, ".v2.f32",
1224  StoreParamScalar2F32>;
1225def StoreParamV2F64  : StoreParamVecInst<V2F64Regs, ".v2.f64",
1226  StoreParamScalar2F64>;
1227
1228def StoreRetvalV4I32  : StoreRetvalVecInst<V4I32Regs, ".v4.b32",
1229  StoreRetvalScalar4I32>;
1230def StoreRetvalV4I16  : StoreRetvalVecInst<V4I16Regs, ".v4.b16",
1231  StoreRetvalScalar4I16>;
1232def StoreRetvalV4I8   : StoreRetvalVecInst<V4I8Regs,  ".v4.b8",
1233  StoreRetvalScalar4I8>;
1234
1235def StoreRetvalV2I64  : StoreRetvalVecInst<V2I64Regs, ".v2.b64",
1236  StoreRetvalScalar2I64>;
1237def StoreRetvalV2I32  : StoreRetvalVecInst<V2I32Regs, ".v2.b32",
1238  StoreRetvalScalar2I32>;
1239def StoreRetvalV2I16  : StoreRetvalVecInst<V2I16Regs, ".v2.b16",
1240  StoreRetvalScalar2I16>;
1241def StoreRetvalV2I8   : StoreRetvalVecInst<V2I8Regs,  ".v2.b8",
1242  StoreRetvalScalar2I8>;
1243
1244def StoreRetvalV4F32  : StoreRetvalVecInst<V4F32Regs, ".v4.f32",
1245  StoreRetvalScalar4F32>;
1246def StoreRetvalV2F32  : StoreRetvalVecInst<V2F32Regs, ".v2.f32",
1247  StoreRetvalScalar2F32>;
1248def StoreRetvalV2F64  : StoreRetvalVecInst<V2F64Regs, ".v2.f64",
1249  StoreRetvalScalar2F64>;
1250
1251}
1252
1253
1254// Int vector to int scalar bit convert
1255// v4i8 -> i32
1256def : Pat<(i32 (bitconvert V4I8Regs:$s)),
1257          (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1258                     (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3))>;
1259// v4i16 -> i64
1260def : Pat<(i64 (bitconvert V4I16Regs:$s)),
1261          (V4I16toI64 (V4i16Extract V4I16Regs:$s,0),
1262            (V4i16Extract V4I16Regs:$s,1),
1263                     (V4i16Extract V4I16Regs:$s,2),
1264                     (V4i16Extract V4I16Regs:$s,3))>;
1265// v2i8 -> i16
1266def : Pat<(i16 (bitconvert V2I8Regs:$s)),
1267          (V2I8toI16 (V2i8Extract V2I8Regs:$s,0), (V2i8Extract V2I8Regs:$s,1))>;
1268// v2i16 -> i32
1269def : Pat<(i32 (bitconvert V2I16Regs:$s)),
1270          (V2I16toI32 (V2i16Extract V2I16Regs:$s,0),
1271            (V2i16Extract V2I16Regs:$s,1))>;
1272// v2i32 -> i64
1273def : Pat<(i64 (bitconvert V2I32Regs:$s)),
1274          (V2I32toI64 (V2i32Extract V2I32Regs:$s,0),
1275            (V2i32Extract V2I32Regs:$s,1))>;
1276
1277// Int scalar to int vector bit convert
1278let VecInstType=isVecDest.Value in {
1279// i32 -> v4i8
1280def VecI32toV4I8 : NVPTXVecInst<(outs V4I8Regs:$d), (ins Int32Regs:$s),
1281                                "Error!",
1282                                [(set V4I8Regs:$d, (bitconvert Int32Regs:$s))],
1283                                I32toV4I8>;
1284// i64 -> v4i16
1285def VecI64toV4I16 : NVPTXVecInst<(outs V4I16Regs:$d), (ins Int64Regs:$s),
1286                                 "Error!",
1287                                [(set V4I16Regs:$d, (bitconvert Int64Regs:$s))],
1288                                 I64toV4I16>;
1289// i16 -> v2i8
1290def VecI16toV2I8 : NVPTXVecInst<(outs V2I8Regs:$d), (ins Int16Regs:$s),
1291                                "Error!",
1292                               [(set V2I8Regs:$d, (bitconvert Int16Regs:$s))],
1293                                I16toV2I8>;
1294// i32 -> v2i16
1295def VecI32toV2I16 : NVPTXVecInst<(outs V2I16Regs:$d), (ins Int32Regs:$s),
1296                                 "Error!",
1297                                [(set V2I16Regs:$d, (bitconvert Int32Regs:$s))],
1298                                 I32toV2I16>;
1299// i64 -> v2i32
1300def VecI64toV2I32 : NVPTXVecInst<(outs V2I32Regs:$d), (ins Int64Regs:$s),
1301                                  "Error!",
1302                                [(set V2I32Regs:$d, (bitconvert Int64Regs:$s))],
1303                                  I64toV2I32>;
1304}
1305
1306// Int vector to int vector bit convert
1307// v4i8 -> v2i16
1308def : Pat<(v2i16 (bitconvert V4I8Regs:$s)),
1309          (VecI32toV2I16
1310          (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1311                    (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3)))>;
1312// v4i16 -> v2i32
1313def : Pat<(v2i32 (bitconvert V4I16Regs:$s)),
1314          (VecI64toV2I32
1315       (V4I16toI64 (V4i16Extract V4I16Regs:$s,0), (V4i16Extract V4I16Regs:$s,1),
1316                (V4i16Extract V4I16Regs:$s,2), (V4i16Extract V4I16Regs:$s,3)))>;
1317// v2i16 -> v4i8
1318def : Pat<(v4i8 (bitconvert V2I16Regs:$s)),
1319          (VecI32toV4I8
1320    (V2I16toI32 (V2i16Extract V2I16Regs:$s,0), (V2i16Extract V2I16Regs:$s,1)))>;
1321// v2i32 -> v4i16
1322def : Pat<(v4i16 (bitconvert V2I32Regs:$s)),
1323          (VecI64toV4I16
1324    (V2I32toI64 (V2i32Extract V2I32Regs:$s,0), (V2i32Extract V2I32Regs:$s,1)))>;
1325// v2i64 -> v4i32
1326def : Pat<(v4i32 (bitconvert V2I64Regs:$s)),
1327          (Build_Vector4_i32
1328            (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 0)), 0),
1329            (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 0)), 1),
1330            (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 1)), 0),
1331            (V2i32Extract (VecI64toV2I32 (V2i64Extract V2I64Regs:$s, 1)), 1))>;
1332// v4i32 -> v2i64
1333def : Pat<(v2i64 (bitconvert V4I32Regs:$s)),
1334          (Build_Vector2_i64
1335      (V2I32toI64 (V4i32Extract V4I32Regs:$s,0), (V4i32Extract V4I32Regs:$s,1)),
1336    (V2I32toI64 (V4i32Extract V4I32Regs:$s,2), (V4i32Extract V4I32Regs:$s,3)))>;
1337
1338// Fp scalar to fp vector convert
1339// f64 -> v2f32
1340let VecInstType=isVecDest.Value in {
1341def VecF64toV2F32 : NVPTXVecInst<(outs V2F32Regs:$d), (ins Float64Regs:$s),
1342                                  "Error!",
1343                              [(set V2F32Regs:$d, (bitconvert Float64Regs:$s))],
1344                                  F64toV2F32>;
1345}
1346
1347// Fp vector to fp scalar convert
1348// v2f32 -> f64
1349def : Pat<(f64 (bitconvert V2F32Regs:$s)),
1350     (V2F32toF64 (V2f32Extract V2F32Regs:$s,0), (V2f32Extract V2F32Regs:$s,1))>;
1351
1352// Fp scalar to int vector convert
1353// f32 -> v4i8
1354def : Pat<(v4i8 (bitconvert Float32Regs:$s)),
1355          (VecI32toV4I8 (BITCONVERT_32_F2I Float32Regs:$s))>;
1356// f32 -> v2i16
1357def : Pat<(v2i16 (bitconvert Float32Regs:$s)),
1358          (VecI32toV2I16 (BITCONVERT_32_F2I Float32Regs:$s))>;
1359// f64 -> v4i16
1360def : Pat<(v4i16 (bitconvert Float64Regs:$s)),
1361          (VecI64toV4I16 (BITCONVERT_64_F2I Float64Regs:$s))>;
1362// f64 -> v2i32
1363def : Pat<(v2i32 (bitconvert Float64Regs:$s)),
1364          (VecI64toV2I32 (BITCONVERT_64_F2I Float64Regs:$s))>;
1365
1366// Int vector to fp scalar convert
1367// v4i8 -> f32
1368def : Pat<(f32 (bitconvert V4I8Regs:$s)),
1369          (BITCONVERT_32_I2F
1370          (V4I8toI32 (V4i8Extract V4I8Regs:$s,0), (V4i8Extract V4I8Regs:$s,1),
1371                    (V4i8Extract V4I8Regs:$s,2), (V4i8Extract V4I8Regs:$s,3)))>;
1372// v4i16 -> f64
1373def : Pat<(f64 (bitconvert V4I16Regs:$s)),
1374          (BITCONVERT_64_I2F
1375       (V4I16toI64 (V4i16Extract V4I16Regs:$s,0), (V4i16Extract V4I16Regs:$s,1),
1376                (V4i16Extract V4I16Regs:$s,2), (V4i16Extract V4I16Regs:$s,3)))>;
1377// v2i16 -> f32
1378def : Pat<(f32 (bitconvert V2I16Regs:$s)),
1379          (BITCONVERT_32_I2F
1380    (V2I16toI32 (V2i16Extract V2I16Regs:$s,0), (V2i16Extract V2I16Regs:$s,1)))>;
1381// v2i32 -> f64
1382def : Pat<(f64 (bitconvert V2I32Regs:$s)),
1383          (BITCONVERT_64_I2F
1384    (V2I32toI64 (V2i32Extract V2I32Regs:$s,0), (V2i32Extract V2I32Regs:$s,1)))>;
1385
1386// Int scalar to fp vector convert
1387// i64 -> v2f32
1388def : Pat<(v2f32 (bitconvert Int64Regs:$s)),
1389          (VecF64toV2F32 (BITCONVERT_64_I2F Int64Regs:$s))>;
1390
1391// Fp vector to int scalar convert
1392// v2f32 -> i64
1393def : Pat<(i64 (bitconvert V2F32Regs:$s)),
1394          (BITCONVERT_64_F2I
1395    (V2F32toF64 (V2f32Extract V2F32Regs:$s,0), (V2f32Extract V2F32Regs:$s,1)))>;
1396
1397// Int vector to fp vector convert
1398// v2i64 -> v4f32
1399def : Pat<(v4f32 (bitconvert V2I64Regs:$s)),
1400          (Build_Vector4_f32
1401            (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1402              (V2i64Extract V2I64Regs:$s, 0)), 0)),
1403            (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1404              (V2i64Extract V2I64Regs:$s, 0)), 1)),
1405            (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1406              (V2i64Extract V2I64Regs:$s, 1)), 0)),
1407            (BITCONVERT_32_I2F (V2i32Extract (VecI64toV2I32
1408              (V2i64Extract V2I64Regs:$s, 1)), 1)))>;
1409// v2i64 -> v2f64
1410def : Pat<(v2f64 (bitconvert V2I64Regs:$s)),
1411    (Build_Vector2_f64
1412            (BITCONVERT_64_I2F (V2i64Extract V2I64Regs:$s,0)),
1413            (BITCONVERT_64_I2F (V2i64Extract V2I64Regs:$s,1)))>;
1414// v2i32 -> v2f32
1415def : Pat<(v2f32 (bitconvert V2I32Regs:$s)),
1416    (Build_Vector2_f32
1417            (BITCONVERT_32_I2F (V2i32Extract V2I32Regs:$s,0)),
1418            (BITCONVERT_32_I2F (V2i32Extract V2I32Regs:$s,1)))>;
1419// v4i32 -> v2f64
1420def : Pat<(v2f64 (bitconvert V4I32Regs:$s)),
1421          (Build_Vector2_f64
1422           (BITCONVERT_64_I2F (V2I32toI64 (V4i32Extract V4I32Regs:$s,0),
1423             (V4i32Extract V4I32Regs:$s,1))),
1424           (BITCONVERT_64_I2F (V2I32toI64 (V4i32Extract V4I32Regs:$s,2),
1425             (V4i32Extract V4I32Regs:$s,3))))>;
1426// v4i32 -> v4f32
1427def : Pat<(v4f32 (bitconvert V4I32Regs:$s)),
1428    (Build_Vector4_f32
1429            (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,0)),
1430            (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,1)),
1431            (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,2)),
1432            (BITCONVERT_32_I2F (V4i32Extract V4I32Regs:$s,3)))>;
1433// v4i16 -> v2f32
1434def : Pat<(v2f32 (bitconvert V4I16Regs:$s)),
1435          (VecF64toV2F32 (BITCONVERT_64_I2F
1436          (V4I16toI64 (V4i16Extract V4I16Regs:$s,0),
1437            (V4i16Extract V4I16Regs:$s,1),
1438                      (V4i16Extract V4I16Regs:$s,2),
1439                      (V4i16Extract V4I16Regs:$s,3))))>;
1440
1441// Fp vector to int vector convert
1442// v2i64 <- v4f32
1443def : Pat<(v2i64 (bitconvert V4F32Regs:$s)),
1444          (Build_Vector2_i64
1445           (BITCONVERT_64_F2I (V2F32toF64 (V4f32Extract V4F32Regs:$s,0),
1446             (V4f32Extract V4F32Regs:$s,1))),
1447           (BITCONVERT_64_F2I (V2F32toF64 (V4f32Extract V4F32Regs:$s,2),
1448             (V4f32Extract V4F32Regs:$s,3))))>;
1449// v2i64 <- v2f64
1450def : Pat<(v2i64 (bitconvert V2F64Regs:$s)),
1451    (Build_Vector2_i64
1452            (BITCONVERT_64_F2I (V2f64Extract V2F64Regs:$s,0)),
1453            (BITCONVERT_64_F2I (V2f64Extract V2F64Regs:$s,1)))>;
1454// v2i32 <- v2f32
1455def : Pat<(v2i32 (bitconvert V2F32Regs:$s)),
1456    (Build_Vector2_i32
1457            (BITCONVERT_32_F2I (V2f32Extract V2F32Regs:$s,0)),
1458            (BITCONVERT_32_F2I (V2f32Extract V2F32Regs:$s,1)))>;
1459// v4i32 <- v2f64
1460def : Pat<(v4i32 (bitconvert V2F64Regs:$s)),
1461          (Build_Vector4_i32
1462            (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1463              (V2f64Extract V2F64Regs:$s, 0)), 0)),
1464            (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1465              (V2f64Extract V2F64Regs:$s, 0)), 1)),
1466            (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1467              (V2f64Extract V2F64Regs:$s, 1)), 0)),
1468            (BITCONVERT_32_F2I (V2f32Extract (VecF64toV2F32
1469              (V2f64Extract V2F64Regs:$s, 1)), 1)))>;
1470// v4i32 <- v4f32
1471def : Pat<(v4i32 (bitconvert V4F32Regs:$s)),
1472          (Build_Vector4_i32
1473            (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,0)),
1474            (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,1)),
1475            (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,2)),
1476            (BITCONVERT_32_F2I (V4f32Extract V4F32Regs:$s,3)))>;
1477// v4i16 <- v2f32
1478def : Pat<(v4i16 (bitconvert V2F32Regs:$s)),
1479          (VecI64toV4I16 (BITCONVERT_64_F2I
1480          (V2F32toF64 (V2f32Extract V2F32Regs:$s,0),
1481            (V2f32Extract V2F32Regs:$s,1))))>;
1482