1 use crate::attributes;
2 use crate::builder::Builder;
3 use crate::common::Funclet;
4 use crate::context::CodegenCx;
5 use crate::llvm;
6 use crate::type_::Type;
7 use crate::type_of::LayoutLlvmExt;
8 use crate::value::Value;
9
10 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
11 use rustc_codegen_ssa::mir::operand::OperandValue;
12 use rustc_codegen_ssa::traits::*;
13 use rustc_data_structures::fx::FxHashMap;
14 use rustc_middle::ty::layout::TyAndLayout;
15 use rustc_middle::{bug, span_bug, ty::Instance};
16 use rustc_span::{Pos, Span};
17 use rustc_target::abi::*;
18 use rustc_target::asm::*;
19
20 use libc::{c_char, c_uint};
21 use smallvec::SmallVec;
22
23 impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
codegen_inline_asm( &mut self, template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, line_spans: &[Span], instance: Instance<'_>, dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, )24 fn codegen_inline_asm(
25 &mut self,
26 template: &[InlineAsmTemplatePiece],
27 operands: &[InlineAsmOperandRef<'tcx, Self>],
28 options: InlineAsmOptions,
29 line_spans: &[Span],
30 instance: Instance<'_>,
31 dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
32 ) {
33 let asm_arch = self.tcx.sess.asm_arch.unwrap();
34
35 // Collect the types of output operands
36 let mut constraints = vec![];
37 let mut clobbers = vec![];
38 let mut output_types = vec![];
39 let mut op_idx = FxHashMap::default();
40 let mut clobbered_x87 = false;
41 for (idx, op) in operands.iter().enumerate() {
42 match *op {
43 InlineAsmOperandRef::Out { reg, late, place } => {
44 let is_target_supported = |reg_class: InlineAsmRegClass| {
45 for &(_, feature) in reg_class.supported_types(asm_arch) {
46 if let Some(feature) = feature {
47 let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id());
48 if self.tcx.sess.target_features.contains(&feature)
49 || codegen_fn_attrs.target_features.contains(&feature)
50 {
51 return true;
52 }
53 } else {
54 // Register class is unconditionally supported
55 return true;
56 }
57 }
58 false
59 };
60
61 let mut layout = None;
62 let ty = if let Some(ref place) = place {
63 layout = Some(&place.layout);
64 llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)
65 } else if matches!(
66 reg.reg_class(),
67 InlineAsmRegClass::X86(
68 X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
69 )
70 ) {
71 // Special handling for x87/mmx registers: we always
72 // clobber the whole set if one register is marked as
73 // clobbered. This is due to the way LLVM handles the
74 // FP stack in inline assembly.
75 if !clobbered_x87 {
76 clobbered_x87 = true;
77 clobbers.push("~{st}".to_string());
78 for i in 1..=7 {
79 clobbers.push(format!("~{{st({})}}", i));
80 }
81 }
82 continue;
83 } else if !is_target_supported(reg.reg_class())
84 || reg.reg_class().is_clobber_only(asm_arch)
85 {
86 // We turn discarded outputs into clobber constraints
87 // if the target feature needed by the register class is
88 // disabled. This is necessary otherwise LLVM will try
89 // to actually allocate a register for the dummy output.
90 assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_)));
91 clobbers.push(format!("~{}", reg_to_llvm(reg, None)));
92 continue;
93 } else {
94 // If the output is discarded, we don't really care what
95 // type is used. We're just using this to tell LLVM to
96 // reserve the register.
97 dummy_output_type(self.cx, reg.reg_class())
98 };
99 output_types.push(ty);
100 op_idx.insert(idx, constraints.len());
101 let prefix = if late { "=" } else { "=&" };
102 constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout)));
103 }
104 InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
105 let layout = if let Some(ref out_place) = out_place {
106 &out_place.layout
107 } else {
108 // LLVM required tied operands to have the same type,
109 // so we just use the type of the input.
110 &in_value.layout
111 };
112 let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);
113 output_types.push(ty);
114 op_idx.insert(idx, constraints.len());
115 let prefix = if late { "=" } else { "=&" };
116 constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
117 }
118 _ => {}
119 }
120 }
121
122 // Collect input operands
123 let mut inputs = vec![];
124 for (idx, op) in operands.iter().enumerate() {
125 match *op {
126 InlineAsmOperandRef::In { reg, value } => {
127 let llval =
128 llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);
129 inputs.push(llval);
130 op_idx.insert(idx, constraints.len());
131 constraints.push(reg_to_llvm(reg, Some(&value.layout)));
132 }
133 InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
134 let value = llvm_fixup_input(
135 self,
136 in_value.immediate(),
137 reg.reg_class(),
138 &in_value.layout,
139 );
140 inputs.push(value);
141
142 // In the case of fixed registers, we have the choice of
143 // either using a tied operand or duplicating the constraint.
144 // We prefer the latter because it matches the behavior of
145 // Clang.
146 if late && matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
147 constraints.push(reg_to_llvm(reg, Some(&in_value.layout)).to_string());
148 } else {
149 constraints.push(format!("{}", op_idx[&idx]));
150 }
151 }
152 InlineAsmOperandRef::SymFn { instance } => {
153 inputs.push(self.cx.get_fn(instance));
154 op_idx.insert(idx, constraints.len());
155 constraints.push("s".to_string());
156 }
157 InlineAsmOperandRef::SymStatic { def_id } => {
158 inputs.push(self.cx.get_static(def_id));
159 op_idx.insert(idx, constraints.len());
160 constraints.push("s".to_string());
161 }
162 _ => {}
163 }
164 }
165
166 // Build the template string
167 let mut template_str = String::new();
168 for piece in template {
169 match *piece {
170 InlineAsmTemplatePiece::String(ref s) => {
171 if s.contains('$') {
172 for c in s.chars() {
173 if c == '$' {
174 template_str.push_str("$$");
175 } else {
176 template_str.push(c);
177 }
178 }
179 } else {
180 template_str.push_str(s)
181 }
182 }
183 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
184 match operands[operand_idx] {
185 InlineAsmOperandRef::In { reg, .. }
186 | InlineAsmOperandRef::Out { reg, .. }
187 | InlineAsmOperandRef::InOut { reg, .. } => {
188 let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
189 if let Some(modifier) = modifier {
190 template_str.push_str(&format!(
191 "${{{}:{}}}",
192 op_idx[&operand_idx], modifier
193 ));
194 } else {
195 template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx]));
196 }
197 }
198 InlineAsmOperandRef::Const { ref string } => {
199 // Const operands get injected directly into the template
200 template_str.push_str(string);
201 }
202 InlineAsmOperandRef::SymFn { .. }
203 | InlineAsmOperandRef::SymStatic { .. } => {
204 // Only emit the raw symbol name
205 template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
206 }
207 }
208 }
209 }
210 }
211
212 constraints.append(&mut clobbers);
213 if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
214 match asm_arch {
215 InlineAsmArch::AArch64 | InlineAsmArch::Arm => {
216 constraints.push("~{cc}".to_string());
217 }
218 InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
219 constraints.extend_from_slice(&[
220 "~{dirflag}".to_string(),
221 "~{fpsr}".to_string(),
222 "~{flags}".to_string(),
223 ]);
224 }
225 InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
226 constraints.extend_from_slice(&[
227 "~{vtype}".to_string(),
228 "~{vl}".to_string(),
229 "~{vxsat}".to_string(),
230 "~{vxrm}".to_string(),
231 ]);
232 }
233 InlineAsmArch::Avr => {
234 constraints.push("~{sreg}".to_string());
235 }
236 InlineAsmArch::Nvptx64 => {}
237 InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
238 InlineAsmArch::Hexagon => {}
239 InlineAsmArch::LoongArch64 => {
240 constraints.extend_from_slice(&[
241 "~{$fcc0}".to_string(),
242 "~{$fcc1}".to_string(),
243 "~{$fcc2}".to_string(),
244 "~{$fcc3}".to_string(),
245 "~{$fcc4}".to_string(),
246 "~{$fcc5}".to_string(),
247 "~{$fcc6}".to_string(),
248 "~{$fcc7}".to_string(),
249 ]);
250 }
251 InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
252 InlineAsmArch::S390x => {
253 constraints.push("~{cc}".to_string());
254 }
255 InlineAsmArch::SpirV => {}
256 InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
257 InlineAsmArch::Bpf => {}
258 InlineAsmArch::Msp430 => {
259 constraints.push("~{sr}".to_string());
260 }
261 InlineAsmArch::M68k => {
262 constraints.push("~{ccr}".to_string());
263 }
264 }
265 }
266 if !options.contains(InlineAsmOptions::NOMEM) {
267 // This is actually ignored by LLVM, but it's probably best to keep
268 // it just in case. LLVM instead uses the ReadOnly/ReadNone
269 // attributes on the call instruction to optimize.
270 constraints.push("~{memory}".to_string());
271 }
272 let volatile = !options.contains(InlineAsmOptions::PURE);
273 let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
274 let output_type = match &output_types[..] {
275 [] => self.type_void(),
276 [ty] => ty,
277 tys => self.type_struct(tys, false),
278 };
279 let dialect = match asm_arch {
280 InlineAsmArch::X86 | InlineAsmArch::X86_64
281 if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
282 {
283 llvm::AsmDialect::Intel
284 }
285 _ => llvm::AsmDialect::Att,
286 };
287 let result = inline_asm_call(
288 self,
289 &template_str,
290 &constraints.join(","),
291 &inputs,
292 output_type,
293 volatile,
294 alignstack,
295 dialect,
296 line_spans,
297 options.contains(InlineAsmOptions::MAY_UNWIND),
298 dest_catch_funclet,
299 )
300 .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
301
302 let mut attrs = SmallVec::<[_; 2]>::new();
303 if options.contains(InlineAsmOptions::PURE) {
304 if options.contains(InlineAsmOptions::NOMEM) {
305 attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
306 } else if options.contains(InlineAsmOptions::READONLY) {
307 attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
308 }
309 attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
310 } else if options.contains(InlineAsmOptions::NOMEM) {
311 attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
312 } else {
313 // LLVM doesn't have an attribute to represent ReadOnly + SideEffect
314 }
315 attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
316
317 // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
318 if let Some((dest, _, _)) = dest_catch_funclet {
319 self.switch_to_block(dest);
320 }
321
322 // Write results to outputs
323 for (idx, op) in operands.iter().enumerate() {
324 if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
325 | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
326 {
327 let value = if output_types.len() == 1 {
328 result
329 } else {
330 self.extract_value(result, op_idx[&idx] as u64)
331 };
332 let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);
333 OperandValue::Immediate(value).store(self, place);
334 }
335 }
336 }
337 }
338
339 impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> {
codegen_global_asm( &self, template: &[InlineAsmTemplatePiece], operands: &[GlobalAsmOperandRef<'tcx>], options: InlineAsmOptions, _line_spans: &[Span], )340 fn codegen_global_asm(
341 &self,
342 template: &[InlineAsmTemplatePiece],
343 operands: &[GlobalAsmOperandRef<'tcx>],
344 options: InlineAsmOptions,
345 _line_spans: &[Span],
346 ) {
347 let asm_arch = self.tcx.sess.asm_arch.unwrap();
348
349 // Default to Intel syntax on x86
350 let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
351 && !options.contains(InlineAsmOptions::ATT_SYNTAX);
352
353 // Build the template string
354 let mut template_str = String::new();
355 if intel_syntax {
356 template_str.push_str(".intel_syntax\n");
357 }
358 for piece in template {
359 match *piece {
360 InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
361 InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
362 match operands[operand_idx] {
363 GlobalAsmOperandRef::Const { ref string } => {
364 // Const operands get injected directly into the
365 // template. Note that we don't need to escape $
366 // here unlike normal inline assembly.
367 template_str.push_str(string);
368 }
369 GlobalAsmOperandRef::SymFn { instance } => {
370 let llval = self.get_fn(instance);
371 self.add_compiler_used_global(llval);
372 let symbol = llvm::build_string(|s| unsafe {
373 llvm::LLVMRustGetMangledName(llval, s);
374 })
375 .expect("symbol is not valid UTF-8");
376 template_str.push_str(&symbol);
377 }
378 GlobalAsmOperandRef::SymStatic { def_id } => {
379 let llval = self
380 .renamed_statics
381 .borrow()
382 .get(&def_id)
383 .copied()
384 .unwrap_or_else(|| self.get_static(def_id));
385 self.add_compiler_used_global(llval);
386 let symbol = llvm::build_string(|s| unsafe {
387 llvm::LLVMRustGetMangledName(llval, s);
388 })
389 .expect("symbol is not valid UTF-8");
390 template_str.push_str(&symbol);
391 }
392 }
393 }
394 }
395 }
396 if intel_syntax {
397 template_str.push_str("\n.att_syntax\n");
398 }
399
400 unsafe {
401 llvm::LLVMAppendModuleInlineAsm(
402 self.llmod,
403 template_str.as_ptr().cast(),
404 template_str.len(),
405 );
406 }
407 }
408 }
409
inline_asm_call<'ll>( bx: &mut Builder<'_, 'll, '_>, asm: &str, cons: &str, inputs: &[&'ll Value], output: &'ll llvm::Type, volatile: bool, alignstack: bool, dia: llvm::AsmDialect, line_spans: &[Span], unwind: bool, dest_catch_funclet: Option<( &'ll llvm::BasicBlock, &'ll llvm::BasicBlock, Option<&Funclet<'ll>>, )>, ) -> Option<&'ll Value>410 pub(crate) fn inline_asm_call<'ll>(
411 bx: &mut Builder<'_, 'll, '_>,
412 asm: &str,
413 cons: &str,
414 inputs: &[&'ll Value],
415 output: &'ll llvm::Type,
416 volatile: bool,
417 alignstack: bool,
418 dia: llvm::AsmDialect,
419 line_spans: &[Span],
420 unwind: bool,
421 dest_catch_funclet: Option<(
422 &'ll llvm::BasicBlock,
423 &'ll llvm::BasicBlock,
424 Option<&Funclet<'ll>>,
425 )>,
426 ) -> Option<&'ll Value> {
427 let volatile = if volatile { llvm::True } else { llvm::False };
428 let alignstack = if alignstack { llvm::True } else { llvm::False };
429 let can_throw = if unwind { llvm::True } else { llvm::False };
430
431 let argtys = inputs
432 .iter()
433 .map(|v| {
434 debug!("Asm Input Type: {:?}", *v);
435 bx.cx.val_ty(*v)
436 })
437 .collect::<Vec<_>>();
438
439 debug!("Asm Output Type: {:?}", output);
440 let fty = bx.cx.type_func(&argtys, output);
441 unsafe {
442 // Ask LLVM to verify that the constraints are well-formed.
443 let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len());
444 debug!("constraint verification result: {:?}", constraints_ok);
445 if constraints_ok {
446 let v = llvm::LLVMRustInlineAsm(
447 fty,
448 asm.as_ptr().cast(),
449 asm.len(),
450 cons.as_ptr().cast(),
451 cons.len(),
452 volatile,
453 alignstack,
454 dia,
455 can_throw,
456 );
457
458 let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
459 bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
460 } else {
461 bx.call(fty, None, None, v, inputs, None)
462 };
463
464 // Store mark in a metadata node so we can map LLVM errors
465 // back to source locations. See #17552.
466 let key = "srcloc";
467 let kind = llvm::LLVMGetMDKindIDInContext(
468 bx.llcx,
469 key.as_ptr() as *const c_char,
470 key.len() as c_uint,
471 );
472
473 // srcloc contains one integer for each line of assembly code.
474 // Unfortunately this isn't enough to encode a full span so instead
475 // we just encode the start position of each line.
476 // FIXME: Figure out a way to pass the entire line spans.
477 let mut srcloc = vec![];
478 if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
479 // LLVM inserts an extra line to add the ".intel_syntax", so add
480 // a dummy srcloc entry for it.
481 //
482 // Don't do this if we only have 1 line span since that may be
483 // due to the asm template string coming from a macro. LLVM will
484 // default to the first srcloc for lines that don't have an
485 // associated srcloc.
486 srcloc.push(bx.const_i32(0));
487 }
488 srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32)));
489 let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32);
490 llvm::LLVMSetMetadata(call, kind, md);
491
492 Some(call)
493 } else {
494 // LLVM has detected an issue with our constraints, bail out
495 None
496 }
497 }
498 }
499
500 /// If the register is an xmm/ymm/zmm register then return its index.
xmm_reg_index(reg: InlineAsmReg) -> Option<u32>501 fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
502 match reg {
503 InlineAsmReg::X86(reg)
504 if reg as u32 >= X86InlineAsmReg::xmm0 as u32
505 && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
506 {
507 Some(reg as u32 - X86InlineAsmReg::xmm0 as u32)
508 }
509 InlineAsmReg::X86(reg)
510 if reg as u32 >= X86InlineAsmReg::ymm0 as u32
511 && reg as u32 <= X86InlineAsmReg::ymm15 as u32 =>
512 {
513 Some(reg as u32 - X86InlineAsmReg::ymm0 as u32)
514 }
515 InlineAsmReg::X86(reg)
516 if reg as u32 >= X86InlineAsmReg::zmm0 as u32
517 && reg as u32 <= X86InlineAsmReg::zmm31 as u32 =>
518 {
519 Some(reg as u32 - X86InlineAsmReg::zmm0 as u32)
520 }
521 _ => None,
522 }
523 }
524
525 /// If the register is an AArch64 integer register then return its index.
a64_reg_index(reg: InlineAsmReg) -> Option<u32>526 fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
527 match reg {
528 InlineAsmReg::AArch64(AArch64InlineAsmReg::x0) => Some(0),
529 InlineAsmReg::AArch64(AArch64InlineAsmReg::x1) => Some(1),
530 InlineAsmReg::AArch64(AArch64InlineAsmReg::x2) => Some(2),
531 InlineAsmReg::AArch64(AArch64InlineAsmReg::x3) => Some(3),
532 InlineAsmReg::AArch64(AArch64InlineAsmReg::x4) => Some(4),
533 InlineAsmReg::AArch64(AArch64InlineAsmReg::x5) => Some(5),
534 InlineAsmReg::AArch64(AArch64InlineAsmReg::x6) => Some(6),
535 InlineAsmReg::AArch64(AArch64InlineAsmReg::x7) => Some(7),
536 InlineAsmReg::AArch64(AArch64InlineAsmReg::x8) => Some(8),
537 InlineAsmReg::AArch64(AArch64InlineAsmReg::x9) => Some(9),
538 InlineAsmReg::AArch64(AArch64InlineAsmReg::x10) => Some(10),
539 InlineAsmReg::AArch64(AArch64InlineAsmReg::x11) => Some(11),
540 InlineAsmReg::AArch64(AArch64InlineAsmReg::x12) => Some(12),
541 InlineAsmReg::AArch64(AArch64InlineAsmReg::x13) => Some(13),
542 InlineAsmReg::AArch64(AArch64InlineAsmReg::x14) => Some(14),
543 InlineAsmReg::AArch64(AArch64InlineAsmReg::x15) => Some(15),
544 InlineAsmReg::AArch64(AArch64InlineAsmReg::x16) => Some(16),
545 InlineAsmReg::AArch64(AArch64InlineAsmReg::x17) => Some(17),
546 InlineAsmReg::AArch64(AArch64InlineAsmReg::x18) => Some(18),
547 // x19 is reserved
548 InlineAsmReg::AArch64(AArch64InlineAsmReg::x20) => Some(20),
549 InlineAsmReg::AArch64(AArch64InlineAsmReg::x21) => Some(21),
550 InlineAsmReg::AArch64(AArch64InlineAsmReg::x22) => Some(22),
551 InlineAsmReg::AArch64(AArch64InlineAsmReg::x23) => Some(23),
552 InlineAsmReg::AArch64(AArch64InlineAsmReg::x24) => Some(24),
553 InlineAsmReg::AArch64(AArch64InlineAsmReg::x25) => Some(25),
554 InlineAsmReg::AArch64(AArch64InlineAsmReg::x26) => Some(26),
555 InlineAsmReg::AArch64(AArch64InlineAsmReg::x27) => Some(27),
556 InlineAsmReg::AArch64(AArch64InlineAsmReg::x28) => Some(28),
557 // x29 is reserved
558 InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) => Some(30),
559 _ => None,
560 }
561 }
562
563 /// If the register is an AArch64 vector register then return its index.
a64_vreg_index(reg: InlineAsmReg) -> Option<u32>564 fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
565 match reg {
566 InlineAsmReg::AArch64(reg)
567 if reg as u32 >= AArch64InlineAsmReg::v0 as u32
568 && reg as u32 <= AArch64InlineAsmReg::v31 as u32 =>
569 {
570 Some(reg as u32 - AArch64InlineAsmReg::v0 as u32)
571 }
572 _ => None,
573 }
574 }
575
576 /// Converts a register class to an LLVM constraint code.
reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String577 fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
578 match reg {
579 // For vector registers LLVM wants the register name to match the type size.
580 InlineAsmRegOrRegClass::Reg(reg) => {
581 if let Some(idx) = xmm_reg_index(reg) {
582 let class = if let Some(layout) = layout {
583 match layout.size.bytes() {
584 64 => 'z',
585 32 => 'y',
586 _ => 'x',
587 }
588 } else {
589 // We use f32 as the type for discarded outputs
590 'x'
591 };
592 format!("{{{}mm{}}}", class, idx)
593 } else if let Some(idx) = a64_reg_index(reg) {
594 let class = if let Some(layout) = layout {
595 match layout.size.bytes() {
596 8 => 'x',
597 _ => 'w',
598 }
599 } else {
600 // We use i32 as the type for discarded outputs
601 'w'
602 };
603 if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
604 // LLVM doesn't recognize x30. use lr instead.
605 "{lr}".to_string()
606 } else {
607 format!("{{{}{}}}", class, idx)
608 }
609 } else if let Some(idx) = a64_vreg_index(reg) {
610 let class = if let Some(layout) = layout {
611 match layout.size.bytes() {
612 16 => 'q',
613 8 => 'd',
614 4 => 's',
615 2 => 'h',
616 1 => 'd', // We fixup i8 to i8x8
617 _ => unreachable!(),
618 }
619 } else {
620 // We use i64x2 as the type for discarded outputs
621 'q'
622 };
623 format!("{{{}{}}}", class, idx)
624 } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
625 // LLVM doesn't recognize r14
626 "{lr}".to_string()
627 } else {
628 format!("{{{}}}", reg.name())
629 }
630 }
631 // The constraints can be retrieved from
632 // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
633 InlineAsmRegOrRegClass::RegClass(reg) => match reg {
634 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
635 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
636 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
637 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
638 unreachable!("clobber-only")
639 }
640 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
641 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
642 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
643 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
644 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
645 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
646 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
647 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
648 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w",
649 InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r",
650 InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
651 InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
652 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r",
653 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f",
654 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
655 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
656 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
657 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
658 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
659 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
660 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
661 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
662 unreachable!("clobber-only")
663 }
664 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
665 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
666 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
667 unreachable!("clobber-only")
668 }
669 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
670 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
671 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q",
672 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
673 | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
674 InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
675 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk",
676 InlineAsmRegClass::X86(
677 X86InlineAsmRegClass::x87_reg
678 | X86InlineAsmRegClass::mmx_reg
679 | X86InlineAsmRegClass::kreg0
680 | X86InlineAsmRegClass::tmm_reg,
681 ) => unreachable!("clobber-only"),
682 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
683 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
684 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
685 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r",
686 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d",
687 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r",
688 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w",
689 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
690 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
691 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
692 InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
693 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r",
694 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a",
695 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d",
696 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
697 bug!("LLVM backend does not support SPIR-V")
698 }
699 InlineAsmRegClass::Err => unreachable!(),
700 }
701 .to_string(),
702 }
703 }
704
705 /// Converts a modifier into LLVM's equivalent modifier.
modifier_to_llvm( arch: InlineAsmArch, reg: InlineAsmRegClass, modifier: Option<char>, ) -> Option<char>706 fn modifier_to_llvm(
707 arch: InlineAsmArch,
708 reg: InlineAsmRegClass,
709 modifier: Option<char>,
710 ) -> Option<char> {
711 // The modifiers can be retrieved from
712 // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
713 match reg {
714 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
715 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
716 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
717 if modifier == Some('v') { None } else { modifier }
718 }
719 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
720 unreachable!("clobber-only")
721 }
722 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
723 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
724 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
725 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
726 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
727 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
728 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
729 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
730 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
731 if modifier.is_none() {
732 Some('q')
733 } else {
734 modifier
735 }
736 }
737 InlineAsmRegClass::Hexagon(_) => None,
738 InlineAsmRegClass::LoongArch(_) => None,
739 InlineAsmRegClass::Mips(_) => None,
740 InlineAsmRegClass::Nvptx(_) => None,
741 InlineAsmRegClass::PowerPC(_) => None,
742 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
743 | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
744 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
745 unreachable!("clobber-only")
746 }
747 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
748 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
749 None if arch == InlineAsmArch::X86_64 => Some('q'),
750 None => Some('k'),
751 Some('l') => Some('b'),
752 Some('h') => Some('h'),
753 Some('x') => Some('w'),
754 Some('e') => Some('k'),
755 Some('r') => Some('q'),
756 _ => unreachable!(),
757 },
758 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None,
759 InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
760 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
761 | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
762 (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
763 (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
764 (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
765 (_, Some('x')) => Some('x'),
766 (_, Some('y')) => Some('t'),
767 (_, Some('z')) => Some('g'),
768 _ => unreachable!(),
769 },
770 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
771 InlineAsmRegClass::X86(
772 X86InlineAsmRegClass::x87_reg
773 | X86InlineAsmRegClass::mmx_reg
774 | X86InlineAsmRegClass::kreg0
775 | X86InlineAsmRegClass::tmm_reg,
776 ) => {
777 unreachable!("clobber-only")
778 }
779 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
780 InlineAsmRegClass::Bpf(_) => None,
781 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair)
782 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw)
783 | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
784 Some('h') => Some('B'),
785 Some('l') => Some('A'),
786 _ => None,
787 },
788 InlineAsmRegClass::Avr(_) => None,
789 InlineAsmRegClass::S390x(_) => None,
790 InlineAsmRegClass::Msp430(_) => None,
791 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
792 bug!("LLVM backend does not support SPIR-V")
793 }
794 InlineAsmRegClass::M68k(_) => None,
795 InlineAsmRegClass::Err => unreachable!(),
796 }
797 }
798
799 /// Type to use for outputs that are discarded. It doesn't really matter what
800 /// the type is, as long as it is valid for the constraint code.
dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type801 fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
802 match reg {
803 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
804 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
805 | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
806 cx.type_vector(cx.type_i64(), 2)
807 }
808 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
809 unreachable!("clobber-only")
810 }
811 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
812 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
813 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
814 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
815 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
816 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
817 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
818 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
819 | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
820 cx.type_vector(cx.type_i64(), 2)
821 }
822 InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
823 InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
824 InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
825 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
826 InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
827 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
828 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
829 InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
830 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
831 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
832 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
833 InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr)
834 | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => {
835 unreachable!("clobber-only")
836 }
837 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
838 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
839 InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
840 unreachable!("clobber-only")
841 }
842 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
843 | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
844 InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
845 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
846 | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
847 | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
848 InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
849 InlineAsmRegClass::X86(
850 X86InlineAsmRegClass::x87_reg
851 | X86InlineAsmRegClass::mmx_reg
852 | X86InlineAsmRegClass::kreg0
853 | X86InlineAsmRegClass::tmm_reg,
854 ) => {
855 unreachable!("clobber-only")
856 }
857 InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
858 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
859 InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
860 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
861 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
862 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
863 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
864 InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
865 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(),
866 InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
867 InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
868 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
869 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
870 InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
871 InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
872 bug!("LLVM backend does not support SPIR-V")
873 }
874 InlineAsmRegClass::Err => unreachable!(),
875 }
876 }
877
878 /// Helper function to get the LLVM type for a Scalar. Pointers are returned as
879 /// the equivalent integer type.
llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type880 fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
881 let dl = &cx.tcx.data_layout;
882 match scalar.primitive() {
883 Primitive::Int(Integer::I8, _) => cx.type_i8(),
884 Primitive::Int(Integer::I16, _) => cx.type_i16(),
885 Primitive::Int(Integer::I32, _) => cx.type_i32(),
886 Primitive::Int(Integer::I64, _) => cx.type_i64(),
887 Primitive::F32 => cx.type_f32(),
888 Primitive::F64 => cx.type_f64(),
889 // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
890 Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
891 _ => unreachable!(),
892 }
893 }
894
895 /// Fix up an input value to work around LLVM bugs.
llvm_fixup_input<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, mut value: &'ll Value, reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value896 fn llvm_fixup_input<'ll, 'tcx>(
897 bx: &mut Builder<'_, 'll, 'tcx>,
898 mut value: &'ll Value,
899 reg: InlineAsmRegClass,
900 layout: &TyAndLayout<'tcx>,
901 ) -> &'ll Value {
902 let dl = &bx.tcx.data_layout;
903 match (reg, layout.abi) {
904 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
905 if let Primitive::Int(Integer::I8, _) = s.primitive() {
906 let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
907 bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
908 } else {
909 value
910 }
911 }
912 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
913 let elem_ty = llvm_asm_scalar_type(bx.cx, s);
914 let count = 16 / layout.size.bytes();
915 let vec_ty = bx.cx.type_vector(elem_ty, count);
916 // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
917 if let Primitive::Pointer(_) = s.primitive() {
918 let t = bx.type_from_integer(dl.ptr_sized_integer());
919 value = bx.ptrtoint(value, t);
920 }
921 bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
922 }
923 (
924 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
925 Abi::Vector { element, count },
926 ) if layout.size.bytes() == 8 => {
927 let elem_ty = llvm_asm_scalar_type(bx.cx, element);
928 let vec_ty = bx.cx.type_vector(elem_ty, count);
929 let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
930 bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
931 }
932 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
933 if s.primitive() == Primitive::F64 =>
934 {
935 bx.bitcast(value, bx.cx.type_i64())
936 }
937 (
938 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
939 Abi::Vector { .. },
940 ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
941 (
942 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
943 Abi::Scalar(s),
944 ) => {
945 if let Primitive::Int(Integer::I32, _) = s.primitive() {
946 bx.bitcast(value, bx.cx.type_f32())
947 } else {
948 value
949 }
950 }
951 (
952 InlineAsmRegClass::Arm(
953 ArmInlineAsmRegClass::dreg
954 | ArmInlineAsmRegClass::dreg_low8
955 | ArmInlineAsmRegClass::dreg_low16,
956 ),
957 Abi::Scalar(s),
958 ) => {
959 if let Primitive::Int(Integer::I64, _) = s.primitive() {
960 bx.bitcast(value, bx.cx.type_f64())
961 } else {
962 value
963 }
964 }
965 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
966 match s.primitive() {
967 // MIPS only supports register-length arithmetics.
968 Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
969 Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()),
970 Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()),
971 _ => value,
972 }
973 }
974 _ => value,
975 }
976 }
977
978 /// Fix up an output value to work around LLVM bugs.
llvm_fixup_output<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, mut value: &'ll Value, reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Value979 fn llvm_fixup_output<'ll, 'tcx>(
980 bx: &mut Builder<'_, 'll, 'tcx>,
981 mut value: &'ll Value,
982 reg: InlineAsmRegClass,
983 layout: &TyAndLayout<'tcx>,
984 ) -> &'ll Value {
985 match (reg, layout.abi) {
986 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
987 if let Primitive::Int(Integer::I8, _) = s.primitive() {
988 bx.extract_element(value, bx.const_i32(0))
989 } else {
990 value
991 }
992 }
993 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
994 value = bx.extract_element(value, bx.const_i32(0));
995 if let Primitive::Pointer(_) = s.primitive() {
996 value = bx.inttoptr(value, layout.llvm_type(bx.cx));
997 }
998 value
999 }
1000 (
1001 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
1002 Abi::Vector { element, count },
1003 ) if layout.size.bytes() == 8 => {
1004 let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1005 let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1006 let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1007 bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1008 }
1009 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
1010 if s.primitive() == Primitive::F64 =>
1011 {
1012 bx.bitcast(value, bx.cx.type_f64())
1013 }
1014 (
1015 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1016 Abi::Vector { .. },
1017 ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1018 (
1019 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1020 Abi::Scalar(s),
1021 ) => {
1022 if let Primitive::Int(Integer::I32, _) = s.primitive() {
1023 bx.bitcast(value, bx.cx.type_i32())
1024 } else {
1025 value
1026 }
1027 }
1028 (
1029 InlineAsmRegClass::Arm(
1030 ArmInlineAsmRegClass::dreg
1031 | ArmInlineAsmRegClass::dreg_low8
1032 | ArmInlineAsmRegClass::dreg_low16,
1033 ),
1034 Abi::Scalar(s),
1035 ) => {
1036 if let Primitive::Int(Integer::I64, _) = s.primitive() {
1037 bx.bitcast(value, bx.cx.type_i64())
1038 } else {
1039 value
1040 }
1041 }
1042 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
1043 match s.primitive() {
1044 // MIPS only supports register-length arithmetics.
1045 Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1046 Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1047 Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()),
1048 Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()),
1049 _ => value,
1050 }
1051 }
1052 _ => value,
1053 }
1054 }
1055
1056 /// Output type to use for llvm_fixup_output.
llvm_fixup_output_type<'ll, 'tcx>( cx: &CodegenCx<'ll, 'tcx>, reg: InlineAsmRegClass, layout: &TyAndLayout<'tcx>, ) -> &'ll Type1057 fn llvm_fixup_output_type<'ll, 'tcx>(
1058 cx: &CodegenCx<'ll, 'tcx>,
1059 reg: InlineAsmRegClass,
1060 layout: &TyAndLayout<'tcx>,
1061 ) -> &'ll Type {
1062 match (reg, layout.abi) {
1063 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
1064 if let Primitive::Int(Integer::I8, _) = s.primitive() {
1065 cx.type_vector(cx.type_i8(), 8)
1066 } else {
1067 layout.llvm_type(cx)
1068 }
1069 }
1070 (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => {
1071 let elem_ty = llvm_asm_scalar_type(cx, s);
1072 let count = 16 / layout.size.bytes();
1073 cx.type_vector(elem_ty, count)
1074 }
1075 (
1076 InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16),
1077 Abi::Vector { element, count },
1078 ) if layout.size.bytes() == 8 => {
1079 let elem_ty = llvm_asm_scalar_type(cx, element);
1080 cx.type_vector(elem_ty, count * 2)
1081 }
1082 (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s))
1083 if s.primitive() == Primitive::F64 =>
1084 {
1085 cx.type_i64()
1086 }
1087 (
1088 InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1089 Abi::Vector { .. },
1090 ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1091 (
1092 InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1093 Abi::Scalar(s),
1094 ) => {
1095 if let Primitive::Int(Integer::I32, _) = s.primitive() {
1096 cx.type_f32()
1097 } else {
1098 layout.llvm_type(cx)
1099 }
1100 }
1101 (
1102 InlineAsmRegClass::Arm(
1103 ArmInlineAsmRegClass::dreg
1104 | ArmInlineAsmRegClass::dreg_low8
1105 | ArmInlineAsmRegClass::dreg_low16,
1106 ),
1107 Abi::Scalar(s),
1108 ) => {
1109 if let Primitive::Int(Integer::I64, _) = s.primitive() {
1110 cx.type_f64()
1111 } else {
1112 layout.llvm_type(cx)
1113 }
1114 }
1115 (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => {
1116 match s.primitive() {
1117 // MIPS only supports register-length arithmetics.
1118 Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1119 Primitive::F32 => cx.type_i32(),
1120 Primitive::F64 => cx.type_i64(),
1121 _ => layout.llvm_type(cx),
1122 }
1123 }
1124 _ => layout.llvm_type(cx),
1125 }
1126 }
1127