1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "code_generator_arm_vixl.h"
18 #include "mirror/array-inl.h"
19
20 namespace vixl32 = vixl::aarch32;
21 using namespace vixl32; // NOLINT(build/namespaces)
22
23 namespace art {
24 namespace arm {
25
26 using helpers::DRegisterFrom;
27 using helpers::Int64ConstantFrom;
28 using helpers::InputDRegisterAt;
29 using helpers::InputRegisterAt;
30 using helpers::OutputDRegister;
31 using helpers::OutputRegister;
32 using helpers::RegisterFrom;
33
34 #define __ GetVIXLAssembler()->
35
VisitVecReplicateScalar(HVecReplicateScalar * instruction)36 void LocationsBuilderARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
37 LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
38 switch (instruction->GetPackedType()) {
39 case DataType::Type::kBool:
40 case DataType::Type::kUint8:
41 case DataType::Type::kInt8:
42 case DataType::Type::kUint16:
43 case DataType::Type::kInt16:
44 case DataType::Type::kInt32:
45 locations->SetInAt(0, Location::RequiresRegister());
46 locations->SetOut(Location::RequiresFpuRegister());
47 break;
48 default:
49 LOG(FATAL) << "Unsupported SIMD type";
50 UNREACHABLE();
51 }
52 }
53
VisitVecReplicateScalar(HVecReplicateScalar * instruction)54 void InstructionCodeGeneratorARMVIXL::VisitVecReplicateScalar(HVecReplicateScalar* instruction) {
55 LocationSummary* locations = instruction->GetLocations();
56 vixl32::DRegister dst = DRegisterFrom(locations->Out());
57 switch (instruction->GetPackedType()) {
58 case DataType::Type::kBool:
59 case DataType::Type::kUint8:
60 case DataType::Type::kInt8:
61 DCHECK_EQ(8u, instruction->GetVectorLength());
62 __ Vdup(Untyped8, dst, InputRegisterAt(instruction, 0));
63 break;
64 case DataType::Type::kUint16:
65 case DataType::Type::kInt16:
66 DCHECK_EQ(4u, instruction->GetVectorLength());
67 __ Vdup(Untyped16, dst, InputRegisterAt(instruction, 0));
68 break;
69 case DataType::Type::kInt32:
70 DCHECK_EQ(2u, instruction->GetVectorLength());
71 __ Vdup(Untyped32, dst, InputRegisterAt(instruction, 0));
72 break;
73 default:
74 LOG(FATAL) << "Unsupported SIMD type";
75 UNREACHABLE();
76 }
77 }
78
VisitVecExtractScalar(HVecExtractScalar * instruction)79 void LocationsBuilderARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
80 LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
81 switch (instruction->GetPackedType()) {
82 case DataType::Type::kInt32:
83 locations->SetInAt(0, Location::RequiresFpuRegister());
84 locations->SetOut(Location::RequiresRegister());
85 break;
86 default:
87 LOG(FATAL) << "Unsupported SIMD type";
88 UNREACHABLE();
89 }
90 }
91
VisitVecExtractScalar(HVecExtractScalar * instruction)92 void InstructionCodeGeneratorARMVIXL::VisitVecExtractScalar(HVecExtractScalar* instruction) {
93 LocationSummary* locations = instruction->GetLocations();
94 vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
95 switch (instruction->GetPackedType()) {
96 case DataType::Type::kInt32:
97 DCHECK_EQ(2u, instruction->GetVectorLength());
98 __ Vmov(OutputRegister(instruction), DRegisterLane(src, 0));
99 break;
100 default:
101 LOG(FATAL) << "Unsupported SIMD type";
102 UNREACHABLE();
103 }
104 }
105
106 // Helper to set up locations for vector unary operations.
CreateVecUnOpLocations(ArenaAllocator * allocator,HVecUnaryOperation * instruction)107 static void CreateVecUnOpLocations(ArenaAllocator* allocator, HVecUnaryOperation* instruction) {
108 LocationSummary* locations = new (allocator) LocationSummary(instruction);
109 switch (instruction->GetPackedType()) {
110 case DataType::Type::kBool:
111 locations->SetInAt(0, Location::RequiresFpuRegister());
112 locations->SetOut(Location::RequiresFpuRegister(),
113 instruction->IsVecNot() ? Location::kOutputOverlap
114 : Location::kNoOutputOverlap);
115 break;
116 case DataType::Type::kUint8:
117 case DataType::Type::kInt8:
118 case DataType::Type::kUint16:
119 case DataType::Type::kInt16:
120 case DataType::Type::kInt32:
121 locations->SetInAt(0, Location::RequiresFpuRegister());
122 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
123 break;
124 default:
125 LOG(FATAL) << "Unsupported SIMD type";
126 UNREACHABLE();
127 }
128 }
129
VisitVecReduce(HVecReduce * instruction)130 void LocationsBuilderARMVIXL::VisitVecReduce(HVecReduce* instruction) {
131 CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
132 }
133
VisitVecReduce(HVecReduce * instruction)134 void InstructionCodeGeneratorARMVIXL::VisitVecReduce(HVecReduce* instruction) {
135 LocationSummary* locations = instruction->GetLocations();
136 vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
137 vixl32::DRegister dst = DRegisterFrom(locations->Out());
138 switch (instruction->GetPackedType()) {
139 case DataType::Type::kInt32:
140 DCHECK_EQ(2u, instruction->GetVectorLength());
141 switch (instruction->GetKind()) {
142 case HVecReduce::kSum:
143 __ Vpadd(DataTypeValue::I32, dst, src, src);
144 break;
145 case HVecReduce::kMin:
146 __ Vpmin(DataTypeValue::S32, dst, src, src);
147 break;
148 case HVecReduce::kMax:
149 __ Vpmax(DataTypeValue::S32, dst, src, src);
150 break;
151 }
152 break;
153 default:
154 LOG(FATAL) << "Unsupported SIMD type";
155 UNREACHABLE();
156 }
157 }
158
VisitVecCnv(HVecCnv * instruction)159 void LocationsBuilderARMVIXL::VisitVecCnv(HVecCnv* instruction) {
160 CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
161 }
162
VisitVecCnv(HVecCnv * instruction)163 void InstructionCodeGeneratorARMVIXL::VisitVecCnv(HVecCnv* instruction) {
164 LOG(FATAL) << "No SIMD for " << instruction->GetId();
165 }
166
VisitVecNeg(HVecNeg * instruction)167 void LocationsBuilderARMVIXL::VisitVecNeg(HVecNeg* instruction) {
168 CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
169 }
170
VisitVecNeg(HVecNeg * instruction)171 void InstructionCodeGeneratorARMVIXL::VisitVecNeg(HVecNeg* instruction) {
172 LocationSummary* locations = instruction->GetLocations();
173 vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
174 vixl32::DRegister dst = DRegisterFrom(locations->Out());
175 switch (instruction->GetPackedType()) {
176 case DataType::Type::kUint8:
177 case DataType::Type::kInt8:
178 DCHECK_EQ(8u, instruction->GetVectorLength());
179 __ Vneg(DataTypeValue::S8, dst, src);
180 break;
181 case DataType::Type::kUint16:
182 case DataType::Type::kInt16:
183 DCHECK_EQ(4u, instruction->GetVectorLength());
184 __ Vneg(DataTypeValue::S16, dst, src);
185 break;
186 case DataType::Type::kInt32:
187 DCHECK_EQ(2u, instruction->GetVectorLength());
188 __ Vneg(DataTypeValue::S32, dst, src);
189 break;
190 default:
191 LOG(FATAL) << "Unsupported SIMD type";
192 UNREACHABLE();
193 }
194 }
195
VisitVecAbs(HVecAbs * instruction)196 void LocationsBuilderARMVIXL::VisitVecAbs(HVecAbs* instruction) {
197 CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
198 }
199
VisitVecAbs(HVecAbs * instruction)200 void InstructionCodeGeneratorARMVIXL::VisitVecAbs(HVecAbs* instruction) {
201 LocationSummary* locations = instruction->GetLocations();
202 vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
203 vixl32::DRegister dst = DRegisterFrom(locations->Out());
204 switch (instruction->GetPackedType()) {
205 case DataType::Type::kInt8:
206 DCHECK_EQ(8u, instruction->GetVectorLength());
207 __ Vabs(DataTypeValue::S8, dst, src);
208 break;
209 case DataType::Type::kInt16:
210 DCHECK_EQ(4u, instruction->GetVectorLength());
211 __ Vabs(DataTypeValue::S16, dst, src);
212 break;
213 case DataType::Type::kInt32:
214 DCHECK_EQ(2u, instruction->GetVectorLength());
215 __ Vabs(DataTypeValue::S32, dst, src);
216 break;
217 default:
218 LOG(FATAL) << "Unsupported SIMD type";
219 UNREACHABLE();
220 }
221 }
222
VisitVecNot(HVecNot * instruction)223 void LocationsBuilderARMVIXL::VisitVecNot(HVecNot* instruction) {
224 CreateVecUnOpLocations(GetGraph()->GetAllocator(), instruction);
225 }
226
VisitVecNot(HVecNot * instruction)227 void InstructionCodeGeneratorARMVIXL::VisitVecNot(HVecNot* instruction) {
228 LocationSummary* locations = instruction->GetLocations();
229 vixl32::DRegister src = DRegisterFrom(locations->InAt(0));
230 vixl32::DRegister dst = DRegisterFrom(locations->Out());
231 switch (instruction->GetPackedType()) {
232 case DataType::Type::kBool: // special case boolean-not
233 DCHECK_EQ(8u, instruction->GetVectorLength());
234 __ Vmov(I8, dst, 1);
235 __ Veor(dst, dst, src);
236 break;
237 case DataType::Type::kUint8:
238 case DataType::Type::kInt8:
239 case DataType::Type::kUint16:
240 case DataType::Type::kInt16:
241 case DataType::Type::kInt32:
242 __ Vmvn(I8, dst, src); // lanes do not matter
243 break;
244 default:
245 LOG(FATAL) << "Unsupported SIMD type";
246 UNREACHABLE();
247 }
248 }
249
250 // Helper to set up locations for vector binary operations.
CreateVecBinOpLocations(ArenaAllocator * allocator,HVecBinaryOperation * instruction)251 static void CreateVecBinOpLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
252 LocationSummary* locations = new (allocator) LocationSummary(instruction);
253 switch (instruction->GetPackedType()) {
254 case DataType::Type::kBool:
255 case DataType::Type::kUint8:
256 case DataType::Type::kInt8:
257 case DataType::Type::kUint16:
258 case DataType::Type::kInt16:
259 case DataType::Type::kInt32:
260 locations->SetInAt(0, Location::RequiresFpuRegister());
261 locations->SetInAt(1, Location::RequiresFpuRegister());
262 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
263 break;
264 default:
265 LOG(FATAL) << "Unsupported SIMD type";
266 UNREACHABLE();
267 }
268 }
269
VisitVecAdd(HVecAdd * instruction)270 void LocationsBuilderARMVIXL::VisitVecAdd(HVecAdd* instruction) {
271 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
272 }
273
VisitVecAdd(HVecAdd * instruction)274 void InstructionCodeGeneratorARMVIXL::VisitVecAdd(HVecAdd* instruction) {
275 LocationSummary* locations = instruction->GetLocations();
276 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
277 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
278 vixl32::DRegister dst = DRegisterFrom(locations->Out());
279 switch (instruction->GetPackedType()) {
280 case DataType::Type::kUint8:
281 case DataType::Type::kInt8:
282 DCHECK_EQ(8u, instruction->GetVectorLength());
283 __ Vadd(I8, dst, lhs, rhs);
284 break;
285 case DataType::Type::kUint16:
286 case DataType::Type::kInt16:
287 DCHECK_EQ(4u, instruction->GetVectorLength());
288 __ Vadd(I16, dst, lhs, rhs);
289 break;
290 case DataType::Type::kInt32:
291 DCHECK_EQ(2u, instruction->GetVectorLength());
292 __ Vadd(I32, dst, lhs, rhs);
293 break;
294 default:
295 LOG(FATAL) << "Unsupported SIMD type";
296 UNREACHABLE();
297 }
298 }
299
VisitVecHalvingAdd(HVecHalvingAdd * instruction)300 void LocationsBuilderARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
301 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
302 }
303
VisitVecHalvingAdd(HVecHalvingAdd * instruction)304 void InstructionCodeGeneratorARMVIXL::VisitVecHalvingAdd(HVecHalvingAdd* instruction) {
305 LocationSummary* locations = instruction->GetLocations();
306 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
307 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
308 vixl32::DRegister dst = DRegisterFrom(locations->Out());
309 switch (instruction->GetPackedType()) {
310 case DataType::Type::kUint8:
311 DCHECK_EQ(8u, instruction->GetVectorLength());
312 instruction->IsRounded()
313 ? __ Vrhadd(DataTypeValue::U8, dst, lhs, rhs)
314 : __ Vhadd(DataTypeValue::U8, dst, lhs, rhs);
315 break;
316 case DataType::Type::kInt8:
317 DCHECK_EQ(8u, instruction->GetVectorLength());
318 instruction->IsRounded()
319 ? __ Vrhadd(DataTypeValue::S8, dst, lhs, rhs)
320 : __ Vhadd(DataTypeValue::S8, dst, lhs, rhs);
321 break;
322 case DataType::Type::kUint16:
323 DCHECK_EQ(4u, instruction->GetVectorLength());
324 instruction->IsRounded()
325 ? __ Vrhadd(DataTypeValue::U16, dst, lhs, rhs)
326 : __ Vhadd(DataTypeValue::U16, dst, lhs, rhs);
327 break;
328 case DataType::Type::kInt16:
329 DCHECK_EQ(4u, instruction->GetVectorLength());
330 instruction->IsRounded()
331 ? __ Vrhadd(DataTypeValue::S16, dst, lhs, rhs)
332 : __ Vhadd(DataTypeValue::S16, dst, lhs, rhs);
333 break;
334 default:
335 LOG(FATAL) << "Unsupported SIMD type";
336 UNREACHABLE();
337 }
338 }
339
VisitVecSub(HVecSub * instruction)340 void LocationsBuilderARMVIXL::VisitVecSub(HVecSub* instruction) {
341 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
342 }
343
VisitVecSub(HVecSub * instruction)344 void InstructionCodeGeneratorARMVIXL::VisitVecSub(HVecSub* instruction) {
345 LocationSummary* locations = instruction->GetLocations();
346 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
347 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
348 vixl32::DRegister dst = DRegisterFrom(locations->Out());
349 switch (instruction->GetPackedType()) {
350 case DataType::Type::kUint8:
351 case DataType::Type::kInt8:
352 DCHECK_EQ(8u, instruction->GetVectorLength());
353 __ Vsub(I8, dst, lhs, rhs);
354 break;
355 case DataType::Type::kUint16:
356 case DataType::Type::kInt16:
357 DCHECK_EQ(4u, instruction->GetVectorLength());
358 __ Vsub(I16, dst, lhs, rhs);
359 break;
360 case DataType::Type::kInt32:
361 DCHECK_EQ(2u, instruction->GetVectorLength());
362 __ Vsub(I32, dst, lhs, rhs);
363 break;
364 default:
365 LOG(FATAL) << "Unsupported SIMD type";
366 UNREACHABLE();
367 }
368 }
369
VisitVecMul(HVecMul * instruction)370 void LocationsBuilderARMVIXL::VisitVecMul(HVecMul* instruction) {
371 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
372 }
373
VisitVecMul(HVecMul * instruction)374 void InstructionCodeGeneratorARMVIXL::VisitVecMul(HVecMul* instruction) {
375 LocationSummary* locations = instruction->GetLocations();
376 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
377 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
378 vixl32::DRegister dst = DRegisterFrom(locations->Out());
379 switch (instruction->GetPackedType()) {
380 case DataType::Type::kUint8:
381 case DataType::Type::kInt8:
382 DCHECK_EQ(8u, instruction->GetVectorLength());
383 __ Vmul(I8, dst, lhs, rhs);
384 break;
385 case DataType::Type::kUint16:
386 case DataType::Type::kInt16:
387 DCHECK_EQ(4u, instruction->GetVectorLength());
388 __ Vmul(I16, dst, lhs, rhs);
389 break;
390 case DataType::Type::kInt32:
391 DCHECK_EQ(2u, instruction->GetVectorLength());
392 __ Vmul(I32, dst, lhs, rhs);
393 break;
394 default:
395 LOG(FATAL) << "Unsupported SIMD type";
396 UNREACHABLE();
397 }
398 }
399
VisitVecDiv(HVecDiv * instruction)400 void LocationsBuilderARMVIXL::VisitVecDiv(HVecDiv* instruction) {
401 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
402 }
403
VisitVecDiv(HVecDiv * instruction)404 void InstructionCodeGeneratorARMVIXL::VisitVecDiv(HVecDiv* instruction) {
405 LOG(FATAL) << "No SIMD for " << instruction->GetId();
406 }
407
VisitVecMin(HVecMin * instruction)408 void LocationsBuilderARMVIXL::VisitVecMin(HVecMin* instruction) {
409 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
410 }
411
VisitVecMin(HVecMin * instruction)412 void InstructionCodeGeneratorARMVIXL::VisitVecMin(HVecMin* instruction) {
413 LocationSummary* locations = instruction->GetLocations();
414 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
415 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
416 vixl32::DRegister dst = DRegisterFrom(locations->Out());
417 switch (instruction->GetPackedType()) {
418 case DataType::Type::kUint8:
419 DCHECK_EQ(8u, instruction->GetVectorLength());
420 __ Vmin(DataTypeValue::U8, dst, lhs, rhs);
421 break;
422 case DataType::Type::kInt8:
423 DCHECK_EQ(8u, instruction->GetVectorLength());
424 __ Vmin(DataTypeValue::S8, dst, lhs, rhs);
425 break;
426 case DataType::Type::kUint16:
427 DCHECK_EQ(4u, instruction->GetVectorLength());
428 __ Vmin(DataTypeValue::U16, dst, lhs, rhs);
429 break;
430 case DataType::Type::kInt16:
431 DCHECK_EQ(4u, instruction->GetVectorLength());
432 __ Vmin(DataTypeValue::S16, dst, lhs, rhs);
433 break;
434 case DataType::Type::kUint32:
435 DCHECK_EQ(2u, instruction->GetVectorLength());
436 __ Vmin(DataTypeValue::U32, dst, lhs, rhs);
437 break;
438 case DataType::Type::kInt32:
439 DCHECK_EQ(2u, instruction->GetVectorLength());
440 __ Vmin(DataTypeValue::S32, dst, lhs, rhs);
441 break;
442 default:
443 LOG(FATAL) << "Unsupported SIMD type";
444 UNREACHABLE();
445 }
446 }
447
VisitVecMax(HVecMax * instruction)448 void LocationsBuilderARMVIXL::VisitVecMax(HVecMax* instruction) {
449 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
450 }
451
VisitVecMax(HVecMax * instruction)452 void InstructionCodeGeneratorARMVIXL::VisitVecMax(HVecMax* instruction) {
453 LocationSummary* locations = instruction->GetLocations();
454 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
455 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
456 vixl32::DRegister dst = DRegisterFrom(locations->Out());
457 switch (instruction->GetPackedType()) {
458 case DataType::Type::kUint8:
459 DCHECK_EQ(8u, instruction->GetVectorLength());
460 __ Vmax(DataTypeValue::U8, dst, lhs, rhs);
461 break;
462 case DataType::Type::kInt8:
463 DCHECK_EQ(8u, instruction->GetVectorLength());
464 __ Vmax(DataTypeValue::S8, dst, lhs, rhs);
465 break;
466 case DataType::Type::kUint16:
467 DCHECK_EQ(4u, instruction->GetVectorLength());
468 __ Vmax(DataTypeValue::U16, dst, lhs, rhs);
469 break;
470 case DataType::Type::kInt16:
471 DCHECK_EQ(4u, instruction->GetVectorLength());
472 __ Vmax(DataTypeValue::S16, dst, lhs, rhs);
473 break;
474 case DataType::Type::kUint32:
475 DCHECK_EQ(2u, instruction->GetVectorLength());
476 __ Vmax(DataTypeValue::U32, dst, lhs, rhs);
477 break;
478 case DataType::Type::kInt32:
479 DCHECK_EQ(2u, instruction->GetVectorLength());
480 __ Vmax(DataTypeValue::S32, dst, lhs, rhs);
481 break;
482 default:
483 LOG(FATAL) << "Unsupported SIMD type";
484 UNREACHABLE();
485 }
486 }
487
VisitVecAnd(HVecAnd * instruction)488 void LocationsBuilderARMVIXL::VisitVecAnd(HVecAnd* instruction) {
489 // TODO: Allow constants supported by VAND (immediate).
490 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
491 }
492
VisitVecAnd(HVecAnd * instruction)493 void InstructionCodeGeneratorARMVIXL::VisitVecAnd(HVecAnd* instruction) {
494 LocationSummary* locations = instruction->GetLocations();
495 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
496 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
497 vixl32::DRegister dst = DRegisterFrom(locations->Out());
498 switch (instruction->GetPackedType()) {
499 case DataType::Type::kBool:
500 case DataType::Type::kUint8:
501 case DataType::Type::kInt8:
502 case DataType::Type::kUint16:
503 case DataType::Type::kInt16:
504 case DataType::Type::kInt32:
505 __ Vand(I8, dst, lhs, rhs);
506 break;
507 default:
508 LOG(FATAL) << "Unsupported SIMD type";
509 UNREACHABLE();
510 }
511 }
512
VisitVecAndNot(HVecAndNot * instruction)513 void LocationsBuilderARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
514 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
515 }
516
VisitVecAndNot(HVecAndNot * instruction)517 void InstructionCodeGeneratorARMVIXL::VisitVecAndNot(HVecAndNot* instruction) {
518 LOG(FATAL) << "No SIMD for " << instruction->GetId();
519 }
520
VisitVecOr(HVecOr * instruction)521 void LocationsBuilderARMVIXL::VisitVecOr(HVecOr* instruction) {
522 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
523 }
524
VisitVecOr(HVecOr * instruction)525 void InstructionCodeGeneratorARMVIXL::VisitVecOr(HVecOr* instruction) {
526 LocationSummary* locations = instruction->GetLocations();
527 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
528 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
529 vixl32::DRegister dst = DRegisterFrom(locations->Out());
530 switch (instruction->GetPackedType()) {
531 case DataType::Type::kBool:
532 case DataType::Type::kUint8:
533 case DataType::Type::kInt8:
534 case DataType::Type::kUint16:
535 case DataType::Type::kInt16:
536 case DataType::Type::kInt32:
537 __ Vorr(I8, dst, lhs, rhs);
538 break;
539 default:
540 LOG(FATAL) << "Unsupported SIMD type";
541 UNREACHABLE();
542 }
543 }
544
VisitVecXor(HVecXor * instruction)545 void LocationsBuilderARMVIXL::VisitVecXor(HVecXor* instruction) {
546 CreateVecBinOpLocations(GetGraph()->GetAllocator(), instruction);
547 }
548
VisitVecXor(HVecXor * instruction)549 void InstructionCodeGeneratorARMVIXL::VisitVecXor(HVecXor* instruction) {
550 LocationSummary* locations = instruction->GetLocations();
551 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
552 vixl32::DRegister rhs = DRegisterFrom(locations->InAt(1));
553 vixl32::DRegister dst = DRegisterFrom(locations->Out());
554 switch (instruction->GetPackedType()) {
555 case DataType::Type::kBool:
556 case DataType::Type::kUint8:
557 case DataType::Type::kInt8:
558 case DataType::Type::kUint16:
559 case DataType::Type::kInt16:
560 case DataType::Type::kInt32:
561 __ Veor(I8, dst, lhs, rhs);
562 break;
563 default:
564 LOG(FATAL) << "Unsupported SIMD type";
565 UNREACHABLE();
566 }
567 }
568
569 // Helper to set up locations for vector shift operations.
CreateVecShiftLocations(ArenaAllocator * allocator,HVecBinaryOperation * instruction)570 static void CreateVecShiftLocations(ArenaAllocator* allocator, HVecBinaryOperation* instruction) {
571 LocationSummary* locations = new (allocator) LocationSummary(instruction);
572 switch (instruction->GetPackedType()) {
573 case DataType::Type::kUint8:
574 case DataType::Type::kInt8:
575 case DataType::Type::kUint16:
576 case DataType::Type::kInt16:
577 case DataType::Type::kInt32:
578 locations->SetInAt(0, Location::RequiresFpuRegister());
579 locations->SetInAt(1, Location::ConstantLocation(instruction->InputAt(1)->AsConstant()));
580 locations->SetOut(Location::RequiresFpuRegister(), Location::kNoOutputOverlap);
581 break;
582 default:
583 LOG(FATAL) << "Unsupported SIMD type";
584 UNREACHABLE();
585 }
586 }
587
VisitVecShl(HVecShl * instruction)588 void LocationsBuilderARMVIXL::VisitVecShl(HVecShl* instruction) {
589 CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
590 }
591
VisitVecShl(HVecShl * instruction)592 void InstructionCodeGeneratorARMVIXL::VisitVecShl(HVecShl* instruction) {
593 LocationSummary* locations = instruction->GetLocations();
594 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
595 vixl32::DRegister dst = DRegisterFrom(locations->Out());
596 int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
597 switch (instruction->GetPackedType()) {
598 case DataType::Type::kUint8:
599 case DataType::Type::kInt8:
600 DCHECK_EQ(8u, instruction->GetVectorLength());
601 __ Vshl(I8, dst, lhs, value);
602 break;
603 case DataType::Type::kUint16:
604 case DataType::Type::kInt16:
605 DCHECK_EQ(4u, instruction->GetVectorLength());
606 __ Vshl(I16, dst, lhs, value);
607 break;
608 case DataType::Type::kInt32:
609 DCHECK_EQ(2u, instruction->GetVectorLength());
610 __ Vshl(I32, dst, lhs, value);
611 break;
612 default:
613 LOG(FATAL) << "Unsupported SIMD type";
614 UNREACHABLE();
615 }
616 }
617
VisitVecShr(HVecShr * instruction)618 void LocationsBuilderARMVIXL::VisitVecShr(HVecShr* instruction) {
619 CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
620 }
621
VisitVecShr(HVecShr * instruction)622 void InstructionCodeGeneratorARMVIXL::VisitVecShr(HVecShr* instruction) {
623 LocationSummary* locations = instruction->GetLocations();
624 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
625 vixl32::DRegister dst = DRegisterFrom(locations->Out());
626 int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
627 switch (instruction->GetPackedType()) {
628 case DataType::Type::kUint8:
629 case DataType::Type::kInt8:
630 DCHECK_EQ(8u, instruction->GetVectorLength());
631 __ Vshr(DataTypeValue::S8, dst, lhs, value);
632 break;
633 case DataType::Type::kUint16:
634 case DataType::Type::kInt16:
635 DCHECK_EQ(4u, instruction->GetVectorLength());
636 __ Vshr(DataTypeValue::S16, dst, lhs, value);
637 break;
638 case DataType::Type::kInt32:
639 DCHECK_EQ(2u, instruction->GetVectorLength());
640 __ Vshr(DataTypeValue::S32, dst, lhs, value);
641 break;
642 default:
643 LOG(FATAL) << "Unsupported SIMD type";
644 UNREACHABLE();
645 }
646 }
647
VisitVecUShr(HVecUShr * instruction)648 void LocationsBuilderARMVIXL::VisitVecUShr(HVecUShr* instruction) {
649 CreateVecShiftLocations(GetGraph()->GetAllocator(), instruction);
650 }
651
VisitVecUShr(HVecUShr * instruction)652 void InstructionCodeGeneratorARMVIXL::VisitVecUShr(HVecUShr* instruction) {
653 LocationSummary* locations = instruction->GetLocations();
654 vixl32::DRegister lhs = DRegisterFrom(locations->InAt(0));
655 vixl32::DRegister dst = DRegisterFrom(locations->Out());
656 int32_t value = locations->InAt(1).GetConstant()->AsIntConstant()->GetValue();
657 switch (instruction->GetPackedType()) {
658 case DataType::Type::kUint8:
659 case DataType::Type::kInt8:
660 DCHECK_EQ(8u, instruction->GetVectorLength());
661 __ Vshr(DataTypeValue::U8, dst, lhs, value);
662 break;
663 case DataType::Type::kUint16:
664 case DataType::Type::kInt16:
665 DCHECK_EQ(4u, instruction->GetVectorLength());
666 __ Vshr(DataTypeValue::U16, dst, lhs, value);
667 break;
668 case DataType::Type::kInt32:
669 DCHECK_EQ(2u, instruction->GetVectorLength());
670 __ Vshr(DataTypeValue::U32, dst, lhs, value);
671 break;
672 default:
673 LOG(FATAL) << "Unsupported SIMD type";
674 UNREACHABLE();
675 }
676 }
677
VisitVecSetScalars(HVecSetScalars * instruction)678 void LocationsBuilderARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
679 LocationSummary* locations = new (GetGraph()->GetAllocator()) LocationSummary(instruction);
680
681 DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented
682
683 HInstruction* input = instruction->InputAt(0);
684 bool is_zero = IsZeroBitPattern(input);
685
686 switch (instruction->GetPackedType()) {
687 case DataType::Type::kInt32:
688 locations->SetInAt(0, is_zero ? Location::ConstantLocation(input->AsConstant())
689 : Location::RequiresRegister());
690 locations->SetOut(Location::RequiresFpuRegister());
691 break;
692 default:
693 LOG(FATAL) << "Unsupported SIMD type";
694 UNREACHABLE();
695 }
696 }
697
VisitVecSetScalars(HVecSetScalars * instruction)698 void InstructionCodeGeneratorARMVIXL::VisitVecSetScalars(HVecSetScalars* instruction) {
699 LocationSummary* locations = instruction->GetLocations();
700 vixl32::DRegister dst = DRegisterFrom(locations->Out());
701
702 DCHECK_EQ(1u, instruction->InputCount()); // only one input currently implemented
703
704 // Zero out all other elements first.
705 __ Vmov(I32, dst, 0);
706
707 // Shorthand for any type of zero.
708 if (IsZeroBitPattern(instruction->InputAt(0))) {
709 return;
710 }
711
712 // Set required elements.
713 switch (instruction->GetPackedType()) {
714 case DataType::Type::kInt32:
715 DCHECK_EQ(2u, instruction->GetVectorLength());
716 __ Vmov(Untyped32, DRegisterLane(dst, 0), InputRegisterAt(instruction, 0));
717 break;
718 default:
719 LOG(FATAL) << "Unsupported SIMD type";
720 UNREACHABLE();
721 }
722 }
723
724 // Helper to set up locations for vector accumulations.
CreateVecAccumLocations(ArenaAllocator * allocator,HVecOperation * instruction)725 static void CreateVecAccumLocations(ArenaAllocator* allocator, HVecOperation* instruction) {
726 LocationSummary* locations = new (allocator) LocationSummary(instruction);
727 switch (instruction->GetPackedType()) {
728 case DataType::Type::kUint8:
729 case DataType::Type::kInt8:
730 case DataType::Type::kUint16:
731 case DataType::Type::kInt16:
732 case DataType::Type::kInt32:
733 case DataType::Type::kInt64:
734 locations->SetInAt(0, Location::RequiresFpuRegister());
735 locations->SetInAt(1, Location::RequiresFpuRegister());
736 locations->SetInAt(2, Location::RequiresFpuRegister());
737 locations->SetOut(Location::SameAsFirstInput());
738 break;
739 default:
740 LOG(FATAL) << "Unsupported SIMD type";
741 UNREACHABLE();
742 }
743 }
744
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instruction)745 void LocationsBuilderARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
746 CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
747 }
748
VisitVecMultiplyAccumulate(HVecMultiplyAccumulate * instruction)749 void InstructionCodeGeneratorARMVIXL::VisitVecMultiplyAccumulate(HVecMultiplyAccumulate* instruction) {
750 LOG(FATAL) << "No SIMD for " << instruction->GetId();
751 }
752
VisitVecSADAccumulate(HVecSADAccumulate * instruction)753 void LocationsBuilderARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
754 CreateVecAccumLocations(GetGraph()->GetAllocator(), instruction);
755 }
756
VisitVecSADAccumulate(HVecSADAccumulate * instruction)757 void InstructionCodeGeneratorARMVIXL::VisitVecSADAccumulate(HVecSADAccumulate* instruction) {
758 LocationSummary* locations = instruction->GetLocations();
759 vixl32::DRegister acc = DRegisterFrom(locations->InAt(0));
760 vixl32::DRegister left = DRegisterFrom(locations->InAt(1));
761 vixl32::DRegister right = DRegisterFrom(locations->InAt(2));
762
763 DCHECK(locations->InAt(0).Equals(locations->Out()));
764
765 // Handle all feasible acc_T += sad(a_S, b_S) type combinations (T x S).
766 HVecOperation* a = instruction->InputAt(1)->AsVecOperation();
767 HVecOperation* b = instruction->InputAt(2)->AsVecOperation();
768 DCHECK_EQ(a->GetPackedType(), b->GetPackedType());
769 switch (a->GetPackedType()) {
770 case DataType::Type::kInt32:
771 DCHECK_EQ(2u, a->GetVectorLength());
772 switch (instruction->GetPackedType()) {
773 case DataType::Type::kInt32: {
774 DCHECK_EQ(2u, instruction->GetVectorLength());
775 UseScratchRegisterScope temps(GetVIXLAssembler());
776 vixl32::DRegister tmp = temps.AcquireD();
777 __ Vsub(DataTypeValue::I32, tmp, left, right);
778 __ Vabs(DataTypeValue::S32, tmp, tmp);
779 __ Vadd(DataTypeValue::I32, acc, acc, tmp);
780 break;
781 }
782 default:
783 LOG(FATAL) << "Unsupported SIMD type";
784 UNREACHABLE();
785 }
786 break;
787 default:
788 LOG(FATAL) << "Unsupported SIMD type";
789 UNREACHABLE();
790 }
791 }
792
793 // Return whether the vector memory access operation is guaranteed to be word-aligned (ARM word
794 // size equals to 4).
IsWordAligned(HVecMemoryOperation * instruction)795 static bool IsWordAligned(HVecMemoryOperation* instruction) {
796 return instruction->GetAlignment().IsAlignedAt(4u);
797 }
798
799 // Helper to set up locations for vector memory operations.
CreateVecMemLocations(ArenaAllocator * allocator,HVecMemoryOperation * instruction,bool is_load)800 static void CreateVecMemLocations(ArenaAllocator* allocator,
801 HVecMemoryOperation* instruction,
802 bool is_load) {
803 LocationSummary* locations = new (allocator) LocationSummary(instruction);
804 switch (instruction->GetPackedType()) {
805 case DataType::Type::kBool:
806 case DataType::Type::kUint8:
807 case DataType::Type::kInt8:
808 case DataType::Type::kUint16:
809 case DataType::Type::kInt16:
810 case DataType::Type::kInt32:
811 locations->SetInAt(0, Location::RequiresRegister());
812 locations->SetInAt(1, Location::RegisterOrConstant(instruction->InputAt(1)));
813 if (is_load) {
814 locations->SetOut(Location::RequiresFpuRegister());
815 } else {
816 locations->SetInAt(2, Location::RequiresFpuRegister());
817 }
818 break;
819 default:
820 LOG(FATAL) << "Unsupported SIMD type";
821 UNREACHABLE();
822 }
823 }
824
825 // Helper to set up locations for vector memory operations. Returns the memory operand and,
826 // if used, sets the output parameter scratch to a temporary register used in this operand,
827 // so that the client can release it right after the memory operand use.
VecAddress(HVecMemoryOperation * instruction,UseScratchRegisterScope * temps_scope,vixl32::Register * scratch)828 MemOperand InstructionCodeGeneratorARMVIXL::VecAddress(
829 HVecMemoryOperation* instruction,
830 UseScratchRegisterScope* temps_scope,
831 /*out*/ vixl32::Register* scratch) {
832 LocationSummary* locations = instruction->GetLocations();
833 vixl32::Register base = InputRegisterAt(instruction, 0);
834
835 Location index = locations->InAt(1);
836 size_t size = DataType::Size(instruction->GetPackedType());
837 uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
838 size_t shift = ComponentSizeShiftWidth(size);
839
840 // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
841 DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
842
843 if (index.IsConstant()) {
844 offset += Int64ConstantFrom(index) << shift;
845 return MemOperand(base, offset);
846 } else {
847 *scratch = temps_scope->Acquire();
848 __ Add(*scratch, base, Operand(RegisterFrom(index), ShiftType::LSL, shift));
849
850 return MemOperand(*scratch, offset);
851 }
852 }
853
VecAddressUnaligned(HVecMemoryOperation * instruction,UseScratchRegisterScope * temps_scope,vixl32::Register * scratch)854 AlignedMemOperand InstructionCodeGeneratorARMVIXL::VecAddressUnaligned(
855 HVecMemoryOperation* instruction,
856 UseScratchRegisterScope* temps_scope,
857 /*out*/ vixl32::Register* scratch) {
858 LocationSummary* locations = instruction->GetLocations();
859 vixl32::Register base = InputRegisterAt(instruction, 0);
860
861 Location index = locations->InAt(1);
862 size_t size = DataType::Size(instruction->GetPackedType());
863 uint32_t offset = mirror::Array::DataOffset(size).Uint32Value();
864 size_t shift = ComponentSizeShiftWidth(size);
865
866 // HIntermediateAddress optimization is only applied for scalar ArrayGet and ArraySet.
867 DCHECK(!instruction->InputAt(0)->IsIntermediateAddress());
868
869 if (index.IsConstant()) {
870 offset += Int64ConstantFrom(index) << shift;
871 __ Add(*scratch, base, offset);
872 } else {
873 *scratch = temps_scope->Acquire();
874 __ Add(*scratch, base, offset);
875 __ Add(*scratch, *scratch, Operand(RegisterFrom(index), ShiftType::LSL, shift));
876 }
877 return AlignedMemOperand(*scratch, kNoAlignment);
878 }
879
VisitVecLoad(HVecLoad * instruction)880 void LocationsBuilderARMVIXL::VisitVecLoad(HVecLoad* instruction) {
881 CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ true);
882 }
883
VisitVecLoad(HVecLoad * instruction)884 void InstructionCodeGeneratorARMVIXL::VisitVecLoad(HVecLoad* instruction) {
885 vixl32::DRegister reg = OutputDRegister(instruction);
886 UseScratchRegisterScope temps(GetVIXLAssembler());
887 vixl32::Register scratch;
888
889 DCHECK(instruction->GetPackedType() != DataType::Type::kUint16 || !instruction->IsStringCharAt());
890
891 switch (instruction->GetPackedType()) {
892 case DataType::Type::kBool:
893 case DataType::Type::kUint8:
894 case DataType::Type::kInt8:
895 DCHECK_EQ(8u, instruction->GetVectorLength());
896 if (IsWordAligned(instruction)) {
897 __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
898 } else {
899 __ Vld1(Untyped8,
900 NeonRegisterList(reg, kMultipleLanes),
901 VecAddressUnaligned(instruction, &temps, &scratch));
902 }
903 break;
904 case DataType::Type::kUint16:
905 case DataType::Type::kInt16:
906 DCHECK_EQ(4u, instruction->GetVectorLength());
907 if (IsWordAligned(instruction)) {
908 __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
909 } else {
910 __ Vld1(Untyped16,
911 NeonRegisterList(reg, kMultipleLanes),
912 VecAddressUnaligned(instruction, &temps, &scratch));
913 }
914 break;
915 case DataType::Type::kInt32:
916 DCHECK_EQ(2u, instruction->GetVectorLength());
917 if (IsWordAligned(instruction)) {
918 __ Vldr(reg, VecAddress(instruction, &temps, &scratch));
919 } else {
920 __ Vld1(Untyped32,
921 NeonRegisterList(reg, kMultipleLanes),
922 VecAddressUnaligned(instruction, &temps, &scratch));
923 }
924 break;
925 default:
926 LOG(FATAL) << "Unsupported SIMD type";
927 UNREACHABLE();
928 }
929 }
930
VisitVecStore(HVecStore * instruction)931 void LocationsBuilderARMVIXL::VisitVecStore(HVecStore* instruction) {
932 CreateVecMemLocations(GetGraph()->GetAllocator(), instruction, /*is_load*/ false);
933 }
934
VisitVecStore(HVecStore * instruction)935 void InstructionCodeGeneratorARMVIXL::VisitVecStore(HVecStore* instruction) {
936 vixl32::DRegister reg = InputDRegisterAt(instruction, 2);
937 UseScratchRegisterScope temps(GetVIXLAssembler());
938 vixl32::Register scratch;
939 switch (instruction->GetPackedType()) {
940 case DataType::Type::kBool:
941 case DataType::Type::kUint8:
942 case DataType::Type::kInt8:
943 DCHECK_EQ(8u, instruction->GetVectorLength());
944 if (IsWordAligned(instruction)) {
945 __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
946 } else {
947 __ Vst1(Untyped8,
948 NeonRegisterList(reg, kMultipleLanes),
949 VecAddressUnaligned(instruction, &temps, &scratch));
950 }
951 break;
952 case DataType::Type::kUint16:
953 case DataType::Type::kInt16:
954 DCHECK_EQ(4u, instruction->GetVectorLength());
955 if (IsWordAligned(instruction)) {
956 __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
957 } else {
958 __ Vst1(Untyped16,
959 NeonRegisterList(reg, kMultipleLanes),
960 VecAddressUnaligned(instruction, &temps, &scratch));
961 }
962 break;
963 case DataType::Type::kInt32:
964 DCHECK_EQ(2u, instruction->GetVectorLength());
965 if (IsWordAligned(instruction)) {
966 __ Vstr(reg, VecAddress(instruction, &temps, &scratch));
967 } else {
968 __ Vst1(Untyped32,
969 NeonRegisterList(reg, kMultipleLanes),
970 VecAddressUnaligned(instruction, &temps, &scratch));
971 }
972 break;
973 default:
974 LOG(FATAL) << "Unsupported SIMD type";
975 UNREACHABLE();
976 }
977 }
978
979 #undef __
980
981 } // namespace arm
982 } // namespace art
983