1 //! Interpret intrinsics, lang items and `extern "C"` wellknown functions which their implementation 2 //! is not available. 3 4 use std::cmp; 5 6 use super::*; 7 8 macro_rules! from_bytes { 9 ($ty:tt, $value:expr) => { 10 ($ty::from_le_bytes(match ($value).try_into() { 11 Ok(x) => x, 12 Err(_) => return Err(MirEvalError::TypeError("mismatched size")), 13 })) 14 }; 15 } 16 17 macro_rules! not_supported { 18 ($x: expr) => { 19 return Err(MirEvalError::NotSupported(format!($x))) 20 }; 21 } 22 23 impl Evaluator<'_> { detect_and_exec_special_function( &mut self, def: FunctionId, args: &[IntervalAndTy], generic_args: &Substitution, locals: &Locals<'_>, destination: Interval, span: MirSpan, ) -> Result<bool>24 pub(super) fn detect_and_exec_special_function( 25 &mut self, 26 def: FunctionId, 27 args: &[IntervalAndTy], 28 generic_args: &Substitution, 29 locals: &Locals<'_>, 30 destination: Interval, 31 span: MirSpan, 32 ) -> Result<bool> { 33 let function_data = self.db.function_data(def); 34 let is_intrinsic = match &function_data.abi { 35 Some(abi) => *abi == Interned::new_str("rust-intrinsic"), 36 None => match def.lookup(self.db.upcast()).container { 37 hir_def::ItemContainerId::ExternBlockId(block) => { 38 let id = block.lookup(self.db.upcast()).id; 39 id.item_tree(self.db.upcast())[id.value].abi.as_deref() 40 == Some("rust-intrinsic") 41 } 42 _ => false, 43 }, 44 }; 45 if is_intrinsic { 46 self.exec_intrinsic( 47 function_data.name.as_text().unwrap_or_default().as_str(), 48 args, 49 generic_args, 50 destination, 51 &locals, 52 span, 53 )?; 54 return Ok(true); 55 } 56 let is_extern_c = match def.lookup(self.db.upcast()).container { 57 hir_def::ItemContainerId::ExternBlockId(block) => { 58 let id = block.lookup(self.db.upcast()).id; 59 id.item_tree(self.db.upcast())[id.value].abi.as_deref() == Some("C") 60 } 61 _ => false, 62 }; 63 if is_extern_c { 64 self.exec_extern_c( 65 function_data.name.as_text().unwrap_or_default().as_str(), 66 args, 67 generic_args, 68 destination, 69 &locals, 70 span, 71 )?; 72 return Ok(true); 73 } 74 let alloc_fn = function_data 75 .attrs 76 .iter() 77 .filter_map(|x| x.path().as_ident()) 78 .filter_map(|x| x.as_str()) 79 .find(|x| { 80 [ 81 "rustc_allocator", 82 "rustc_deallocator", 83 "rustc_reallocator", 84 "rustc_allocator_zeroed", 85 ] 86 .contains(x) 87 }); 88 if let Some(alloc_fn) = alloc_fn { 89 self.exec_alloc_fn(alloc_fn, args, destination)?; 90 return Ok(true); 91 } 92 if let Some(x) = self.detect_lang_function(def) { 93 let arg_bytes = 94 args.iter().map(|x| Ok(x.get(&self)?.to_owned())).collect::<Result<Vec<_>>>()?; 95 let result = self.exec_lang_item(x, generic_args, &arg_bytes, locals, span)?; 96 destination.write_from_bytes(self, &result)?; 97 return Ok(true); 98 } 99 Ok(false) 100 } 101 exec_alloc_fn( &mut self, alloc_fn: &str, args: &[IntervalAndTy], destination: Interval, ) -> Result<()>102 fn exec_alloc_fn( 103 &mut self, 104 alloc_fn: &str, 105 args: &[IntervalAndTy], 106 destination: Interval, 107 ) -> Result<()> { 108 match alloc_fn { 109 "rustc_allocator_zeroed" | "rustc_allocator" => { 110 let [size, align] = args else { 111 return Err(MirEvalError::TypeError("rustc_allocator args are not provided")); 112 }; 113 let size = from_bytes!(usize, size.get(self)?); 114 let align = from_bytes!(usize, align.get(self)?); 115 let result = self.heap_allocate(size, align); 116 destination.write_from_bytes(self, &result.to_bytes())?; 117 } 118 "rustc_deallocator" => { /* no-op for now */ } 119 "rustc_reallocator" => { 120 let [ptr, old_size, align, new_size] = args else { 121 return Err(MirEvalError::TypeError("rustc_allocator args are not provided")); 122 }; 123 let ptr = Address::from_bytes(ptr.get(self)?)?; 124 let old_size = from_bytes!(usize, old_size.get(self)?); 125 let new_size = from_bytes!(usize, new_size.get(self)?); 126 let align = from_bytes!(usize, align.get(self)?); 127 let result = self.heap_allocate(new_size, align); 128 Interval { addr: result, size: old_size } 129 .write_from_interval(self, Interval { addr: ptr, size: old_size })?; 130 destination.write_from_bytes(self, &result.to_bytes())?; 131 } 132 _ => not_supported!("unknown alloc function"), 133 } 134 Ok(()) 135 } 136 detect_lang_function(&self, def: FunctionId) -> Option<LangItem>137 fn detect_lang_function(&self, def: FunctionId) -> Option<LangItem> { 138 use LangItem::*; 139 let candidate = lang_attr(self.db.upcast(), def)?; 140 // We want to execute these functions with special logic 141 if [PanicFmt, BeginPanic, SliceLen, DropInPlace].contains(&candidate) { 142 return Some(candidate); 143 } 144 None 145 } 146 exec_lang_item( &mut self, x: LangItem, generic_args: &Substitution, args: &[Vec<u8>], locals: &Locals<'_>, span: MirSpan, ) -> Result<Vec<u8>>147 fn exec_lang_item( 148 &mut self, 149 x: LangItem, 150 generic_args: &Substitution, 151 args: &[Vec<u8>], 152 locals: &Locals<'_>, 153 span: MirSpan, 154 ) -> Result<Vec<u8>> { 155 use LangItem::*; 156 let mut args = args.iter(); 157 match x { 158 BeginPanic => Err(MirEvalError::Panic("<unknown-panic-payload>".to_string())), 159 PanicFmt => { 160 let message = (|| { 161 let arguments_struct = 162 self.db.lang_item(self.crate_id, LangItem::FormatArguments)?.as_struct()?; 163 let arguments_layout = self 164 .layout_adt(arguments_struct.into(), Substitution::empty(Interner)) 165 .ok()?; 166 let arguments_field_pieces = 167 self.db.struct_data(arguments_struct).variant_data.field(&name![pieces])?; 168 let pieces_offset = arguments_layout 169 .fields 170 .offset(u32::from(arguments_field_pieces.into_raw()) as usize) 171 .bytes_usize(); 172 let ptr_size = self.ptr_size(); 173 let arg = args.next()?; 174 let pieces_array_addr = 175 Address::from_bytes(&arg[pieces_offset..pieces_offset + ptr_size]).ok()?; 176 let pieces_array_len = usize::from_le_bytes( 177 (&arg[pieces_offset + ptr_size..pieces_offset + 2 * ptr_size]) 178 .try_into() 179 .ok()?, 180 ); 181 let mut message = "".to_string(); 182 for i in 0..pieces_array_len { 183 let piece_ptr_addr = pieces_array_addr.offset(2 * i * ptr_size); 184 let piece_addr = 185 Address::from_bytes(self.read_memory(piece_ptr_addr, ptr_size).ok()?) 186 .ok()?; 187 let piece_len = usize::from_le_bytes( 188 self.read_memory(piece_ptr_addr.offset(ptr_size), ptr_size) 189 .ok()? 190 .try_into() 191 .ok()?, 192 ); 193 let piece_data = self.read_memory(piece_addr, piece_len).ok()?; 194 message += &std::string::String::from_utf8_lossy(piece_data); 195 } 196 Some(message) 197 })() 198 .unwrap_or_else(|| "<format-args-evaluation-failed>".to_string()); 199 Err(MirEvalError::Panic(message)) 200 } 201 SliceLen => { 202 let arg = args 203 .next() 204 .ok_or(MirEvalError::TypeError("argument of <[T]>::len() is not provided"))?; 205 let ptr_size = arg.len() / 2; 206 Ok(arg[ptr_size..].into()) 207 } 208 DropInPlace => { 209 let ty = 210 generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)).ok_or( 211 MirEvalError::TypeError( 212 "generic argument of drop_in_place is not provided", 213 ), 214 )?; 215 let arg = args 216 .next() 217 .ok_or(MirEvalError::TypeError("argument of drop_in_place is not provided"))?; 218 self.run_drop_glue_deep( 219 ty.clone(), 220 locals, 221 Address::from_bytes(&arg[0..self.ptr_size()])?, 222 &arg[self.ptr_size()..], 223 span, 224 )?; 225 Ok(vec![]) 226 } 227 x => not_supported!("Executing lang item {x:?}"), 228 } 229 } 230 exec_extern_c( &mut self, as_str: &str, args: &[IntervalAndTy], _generic_args: &Substitution, destination: Interval, locals: &Locals<'_>, _span: MirSpan, ) -> Result<()>231 fn exec_extern_c( 232 &mut self, 233 as_str: &str, 234 args: &[IntervalAndTy], 235 _generic_args: &Substitution, 236 destination: Interval, 237 locals: &Locals<'_>, 238 _span: MirSpan, 239 ) -> Result<()> { 240 match as_str { 241 "memcmp" => { 242 let [ptr1, ptr2, size] = args else { 243 return Err(MirEvalError::TypeError("memcmp args are not provided")); 244 }; 245 let addr1 = Address::from_bytes(ptr1.get(self)?)?; 246 let addr2 = Address::from_bytes(ptr2.get(self)?)?; 247 let size = from_bytes!(usize, size.get(self)?); 248 let slice1 = self.read_memory(addr1, size)?; 249 let slice2 = self.read_memory(addr2, size)?; 250 let r: i128 = match slice1.cmp(slice2) { 251 cmp::Ordering::Less => -1, 252 cmp::Ordering::Equal => 0, 253 cmp::Ordering::Greater => 1, 254 }; 255 destination.write_from_bytes(self, &r.to_le_bytes()[..destination.size]) 256 } 257 "write" => { 258 let [fd, ptr, len] = args else { 259 return Err(MirEvalError::TypeError("libc::write args are not provided")); 260 }; 261 let fd = u128::from_le_bytes(pad16(fd.get(self)?, false)); 262 let interval = Interval { 263 addr: Address::from_bytes(ptr.get(self)?)?, 264 size: from_bytes!(usize, len.get(self)?), 265 }; 266 match fd { 267 1 => { 268 self.write_to_stdout(interval)?; 269 } 270 2 => { 271 self.write_to_stderr(interval)?; 272 } 273 _ => not_supported!("write to arbitrary file descriptor"), 274 } 275 destination.write_from_interval(self, len.interval)?; 276 Ok(()) 277 } 278 "pthread_key_create" => { 279 let key = self.thread_local_storage.create_key(); 280 let Some(arg0) = args.get(0) else { 281 return Err(MirEvalError::TypeError("pthread_key_create arg0 is not provided")); 282 }; 283 let arg0_addr = Address::from_bytes(arg0.get(self)?)?; 284 let key_ty = if let Some((ty, ..)) = arg0.ty.as_reference_or_ptr() { 285 ty 286 } else { 287 return Err(MirEvalError::TypeError( 288 "pthread_key_create arg0 is not a pointer", 289 )); 290 }; 291 let arg0_interval = Interval::new( 292 arg0_addr, 293 self.size_of_sized(key_ty, locals, "pthread_key_create key arg")?, 294 ); 295 arg0_interval.write_from_bytes(self, &key.to_le_bytes()[0..arg0_interval.size])?; 296 // return 0 as success 297 destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; 298 Ok(()) 299 } 300 "pthread_getspecific" => { 301 let Some(arg0) = args.get(0) else { 302 return Err(MirEvalError::TypeError("pthread_getspecific arg0 is not provided")); 303 }; 304 let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]); 305 let value = self.thread_local_storage.get_key(key)?; 306 destination.write_from_bytes(self, &value.to_le_bytes()[0..destination.size])?; 307 Ok(()) 308 } 309 "pthread_setspecific" => { 310 let Some(arg0) = args.get(0) else { 311 return Err(MirEvalError::TypeError("pthread_setspecific arg0 is not provided")); 312 }; 313 let key = from_bytes!(usize, &pad16(arg0.get(self)?, false)[0..8]); 314 let Some(arg1) = args.get(1) else { 315 return Err(MirEvalError::TypeError("pthread_setspecific arg1 is not provided")); 316 }; 317 let value = from_bytes!(u128, pad16(arg1.get(self)?, false)); 318 self.thread_local_storage.set_key(key, value)?; 319 // return 0 as success 320 destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; 321 Ok(()) 322 } 323 "pthread_key_delete" => { 324 // we ignore this currently 325 // return 0 as success 326 destination.write_from_bytes(self, &0u64.to_le_bytes()[0..destination.size])?; 327 Ok(()) 328 } 329 _ => not_supported!("unknown external function {as_str}"), 330 } 331 } 332 exec_intrinsic( &mut self, name: &str, args: &[IntervalAndTy], generic_args: &Substitution, destination: Interval, locals: &Locals<'_>, span: MirSpan, ) -> Result<()>333 fn exec_intrinsic( 334 &mut self, 335 name: &str, 336 args: &[IntervalAndTy], 337 generic_args: &Substitution, 338 destination: Interval, 339 locals: &Locals<'_>, 340 span: MirSpan, 341 ) -> Result<()> { 342 if let Some(name) = name.strip_prefix("atomic_") { 343 return self.exec_atomic_intrinsic(name, args, generic_args, destination, locals, span); 344 } 345 if let Some(name) = name.strip_suffix("f64") { 346 let result = match name { 347 "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs" 348 | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => { 349 let [arg] = args else { 350 return Err(MirEvalError::TypeError("f64 intrinsic signature doesn't match fn (f64) -> f64")); 351 }; 352 let arg = from_bytes!(f64, arg.get(self)?); 353 match name { 354 "sqrt" => arg.sqrt(), 355 "sin" => arg.sin(), 356 "cos" => arg.cos(), 357 "exp" => arg.exp(), 358 "exp2" => arg.exp2(), 359 "log" => arg.ln(), 360 "log10" => arg.log10(), 361 "log2" => arg.log2(), 362 "fabs" => arg.abs(), 363 "floor" => arg.floor(), 364 "ceil" => arg.ceil(), 365 "trunc" => arg.trunc(), 366 // FIXME: these rounds should be different, but only `.round()` is stable now. 367 "rint" => arg.round(), 368 "nearbyint" => arg.round(), 369 "round" => arg.round(), 370 "roundeven" => arg.round(), 371 _ => unreachable!(), 372 } 373 } 374 "pow" | "minnum" | "maxnum" | "copysign" => { 375 let [arg1, arg2] = args else { 376 return Err(MirEvalError::TypeError("f64 intrinsic signature doesn't match fn (f64, f64) -> f64")); 377 }; 378 let arg1 = from_bytes!(f64, arg1.get(self)?); 379 let arg2 = from_bytes!(f64, arg2.get(self)?); 380 match name { 381 "pow" => arg1.powf(arg2), 382 "minnum" => arg1.min(arg2), 383 "maxnum" => arg1.max(arg2), 384 "copysign" => arg1.copysign(arg2), 385 _ => unreachable!(), 386 } 387 } 388 "powi" => { 389 let [arg1, arg2] = args else { 390 return Err(MirEvalError::TypeError("powif64 signature doesn't match fn (f64, i32) -> f64")); 391 }; 392 let arg1 = from_bytes!(f64, arg1.get(self)?); 393 let arg2 = from_bytes!(i32, arg2.get(self)?); 394 arg1.powi(arg2) 395 } 396 "fma" => { 397 let [arg1, arg2, arg3] = args else { 398 return Err(MirEvalError::TypeError("fmaf64 signature doesn't match fn (f64, f64, f64) -> f64")); 399 }; 400 let arg1 = from_bytes!(f64, arg1.get(self)?); 401 let arg2 = from_bytes!(f64, arg2.get(self)?); 402 let arg3 = from_bytes!(f64, arg3.get(self)?); 403 arg1.mul_add(arg2, arg3) 404 } 405 _ => not_supported!("unknown f64 intrinsic {name}"), 406 }; 407 return destination.write_from_bytes(self, &result.to_le_bytes()); 408 } 409 if let Some(name) = name.strip_suffix("f32") { 410 let result = match name { 411 "sqrt" | "sin" | "cos" | "exp" | "exp2" | "log" | "log10" | "log2" | "fabs" 412 | "floor" | "ceil" | "trunc" | "rint" | "nearbyint" | "round" | "roundeven" => { 413 let [arg] = args else { 414 return Err(MirEvalError::TypeError("f32 intrinsic signature doesn't match fn (f32) -> f32")); 415 }; 416 let arg = from_bytes!(f32, arg.get(self)?); 417 match name { 418 "sqrt" => arg.sqrt(), 419 "sin" => arg.sin(), 420 "cos" => arg.cos(), 421 "exp" => arg.exp(), 422 "exp2" => arg.exp2(), 423 "log" => arg.ln(), 424 "log10" => arg.log10(), 425 "log2" => arg.log2(), 426 "fabs" => arg.abs(), 427 "floor" => arg.floor(), 428 "ceil" => arg.ceil(), 429 "trunc" => arg.trunc(), 430 // FIXME: these rounds should be different, but only `.round()` is stable now. 431 "rint" => arg.round(), 432 "nearbyint" => arg.round(), 433 "round" => arg.round(), 434 "roundeven" => arg.round(), 435 _ => unreachable!(), 436 } 437 } 438 "pow" | "minnum" | "maxnum" | "copysign" => { 439 let [arg1, arg2] = args else { 440 return Err(MirEvalError::TypeError("f32 intrinsic signature doesn't match fn (f32, f32) -> f32")); 441 }; 442 let arg1 = from_bytes!(f32, arg1.get(self)?); 443 let arg2 = from_bytes!(f32, arg2.get(self)?); 444 match name { 445 "pow" => arg1.powf(arg2), 446 "minnum" => arg1.min(arg2), 447 "maxnum" => arg1.max(arg2), 448 "copysign" => arg1.copysign(arg2), 449 _ => unreachable!(), 450 } 451 } 452 "powi" => { 453 let [arg1, arg2] = args else { 454 return Err(MirEvalError::TypeError("powif32 signature doesn't match fn (f32, i32) -> f32")); 455 }; 456 let arg1 = from_bytes!(f32, arg1.get(self)?); 457 let arg2 = from_bytes!(i32, arg2.get(self)?); 458 arg1.powi(arg2) 459 } 460 "fma" => { 461 let [arg1, arg2, arg3] = args else { 462 return Err(MirEvalError::TypeError("fmaf32 signature doesn't match fn (f32, f32, f32) -> f32")); 463 }; 464 let arg1 = from_bytes!(f32, arg1.get(self)?); 465 let arg2 = from_bytes!(f32, arg2.get(self)?); 466 let arg3 = from_bytes!(f32, arg3.get(self)?); 467 arg1.mul_add(arg2, arg3) 468 } 469 _ => not_supported!("unknown f32 intrinsic {name}"), 470 }; 471 return destination.write_from_bytes(self, &result.to_le_bytes()); 472 } 473 match name { 474 "size_of" => { 475 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 476 return Err(MirEvalError::TypeError("size_of generic arg is not provided")); 477 }; 478 let size = self.size_of_sized(ty, locals, "size_of arg")?; 479 destination.write_from_bytes(self, &size.to_le_bytes()[0..destination.size]) 480 } 481 "min_align_of" | "pref_align_of" => { 482 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 483 return Err(MirEvalError::TypeError("align_of generic arg is not provided")); 484 }; 485 let align = self.layout(ty)?.align.abi.bytes(); 486 destination.write_from_bytes(self, &align.to_le_bytes()[0..destination.size]) 487 } 488 "needs_drop" => { 489 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 490 return Err(MirEvalError::TypeError("size_of generic arg is not provided")); 491 }; 492 let result = !ty.clone().is_copy(self.db, locals.body.owner); 493 destination.write_from_bytes(self, &[u8::from(result)]) 494 } 495 "ptr_guaranteed_cmp" => { 496 // FIXME: this is wrong for const eval, it should return 2 in some 497 // cases. 498 let [lhs, rhs] = args else { 499 return Err(MirEvalError::TypeError("wrapping_add args are not provided")); 500 }; 501 let ans = lhs.get(self)? == rhs.get(self)?; 502 destination.write_from_bytes(self, &[u8::from(ans)]) 503 } 504 "saturating_add" => { 505 let [lhs, rhs] = args else { 506 return Err(MirEvalError::TypeError("saturating_add args are not provided")); 507 }; 508 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 509 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 510 let ans = lhs.saturating_add(rhs); 511 let bits = destination.size * 8; 512 // FIXME: signed 513 let is_signed = false; 514 let mx: u128 = if is_signed { (1 << (bits - 1)) - 1 } else { (1 << bits) - 1 }; 515 // FIXME: signed 516 let mn: u128 = 0; 517 let ans = cmp::min(mx, cmp::max(mn, ans)); 518 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 519 } 520 "wrapping_add" | "unchecked_add" => { 521 let [lhs, rhs] = args else { 522 return Err(MirEvalError::TypeError("wrapping_add args are not provided")); 523 }; 524 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 525 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 526 let ans = lhs.wrapping_add(rhs); 527 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 528 } 529 "wrapping_sub" | "unchecked_sub" | "ptr_offset_from_unsigned" | "ptr_offset_from" => { 530 let [lhs, rhs] = args else { 531 return Err(MirEvalError::TypeError("wrapping_sub args are not provided")); 532 }; 533 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 534 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 535 let ans = lhs.wrapping_sub(rhs); 536 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 537 } 538 "wrapping_mul" | "unchecked_mul" => { 539 let [lhs, rhs] = args else { 540 return Err(MirEvalError::TypeError("wrapping_mul args are not provided")); 541 }; 542 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 543 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 544 let ans = lhs.wrapping_mul(rhs); 545 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 546 } 547 "unchecked_rem" => { 548 // FIXME: signed 549 let [lhs, rhs] = args else { 550 return Err(MirEvalError::TypeError("unchecked_rem args are not provided")); 551 }; 552 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 553 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 554 let ans = lhs.checked_rem(rhs).ok_or_else(|| { 555 MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned()) 556 })?; 557 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 558 } 559 "unchecked_div" | "exact_div" => { 560 // FIXME: signed 561 let [lhs, rhs] = args else { 562 return Err(MirEvalError::TypeError("unchecked_div args are not provided")); 563 }; 564 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 565 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 566 let ans = lhs.checked_div(rhs).ok_or_else(|| { 567 MirEvalError::UndefinedBehavior("unchecked_rem with bad inputs".to_owned()) 568 })?; 569 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 570 } 571 "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => { 572 let [lhs, rhs] = args else { 573 return Err(MirEvalError::TypeError("const_eval_select args are not provided")); 574 }; 575 let result_ty = TyKind::Tuple( 576 2, 577 Substitution::from_iter(Interner, [lhs.ty.clone(), TyBuilder::bool()]), 578 ) 579 .intern(Interner); 580 let op_size = 581 self.size_of_sized(&lhs.ty, locals, "operand of add_with_overflow")?; 582 let lhs = u128::from_le_bytes(pad16(lhs.get(self)?, false)); 583 let rhs = u128::from_le_bytes(pad16(rhs.get(self)?, false)); 584 let (ans, u128overflow) = match name { 585 "add_with_overflow" => lhs.overflowing_add(rhs), 586 "sub_with_overflow" => lhs.overflowing_sub(rhs), 587 "mul_with_overflow" => lhs.overflowing_mul(rhs), 588 _ => unreachable!(), 589 }; 590 let is_overflow = u128overflow 591 || ans.to_le_bytes()[op_size..].iter().any(|&x| x != 0 && x != 255); 592 let is_overflow = vec![u8::from(is_overflow)]; 593 let layout = self.layout(&result_ty)?; 594 let result = self.make_by_layout( 595 layout.size.bytes_usize(), 596 &layout, 597 None, 598 [ans.to_le_bytes()[0..op_size].to_vec(), is_overflow] 599 .into_iter() 600 .map(IntervalOrOwned::Owned), 601 )?; 602 destination.write_from_bytes(self, &result) 603 } 604 "copy" | "copy_nonoverlapping" => { 605 let [src, dst, offset] = args else { 606 return Err(MirEvalError::TypeError("copy_nonoverlapping args are not provided")); 607 }; 608 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 609 return Err(MirEvalError::TypeError("copy_nonoverlapping generic arg is not provided")); 610 }; 611 let src = Address::from_bytes(src.get(self)?)?; 612 let dst = Address::from_bytes(dst.get(self)?)?; 613 let offset = from_bytes!(usize, offset.get(self)?); 614 let size = self.size_of_sized(ty, locals, "copy_nonoverlapping ptr type")?; 615 let size = offset * size; 616 let src = Interval { addr: src, size }; 617 let dst = Interval { addr: dst, size }; 618 dst.write_from_interval(self, src) 619 } 620 "offset" | "arith_offset" => { 621 let [ptr, offset] = args else { 622 return Err(MirEvalError::TypeError("offset args are not provided")); 623 }; 624 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 625 return Err(MirEvalError::TypeError("offset generic arg is not provided")); 626 }; 627 let ptr = u128::from_le_bytes(pad16(ptr.get(self)?, false)); 628 let offset = u128::from_le_bytes(pad16(offset.get(self)?, false)); 629 let size = self.size_of_sized(ty, locals, "offset ptr type")? as u128; 630 let ans = ptr + offset * size; 631 destination.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]) 632 } 633 "assert_inhabited" | "assert_zero_valid" | "assert_uninit_valid" | "assume" => { 634 // FIXME: We should actually implement these checks 635 Ok(()) 636 } 637 "forget" => { 638 // We don't call any drop glue yet, so there is nothing here 639 Ok(()) 640 } 641 "transmute" => { 642 let [arg] = args else { 643 return Err(MirEvalError::TypeError("trasmute arg is not provided")); 644 }; 645 destination.write_from_interval(self, arg.interval) 646 } 647 "likely" | "unlikely" => { 648 let [arg] = args else { 649 return Err(MirEvalError::TypeError("likely arg is not provided")); 650 }; 651 destination.write_from_interval(self, arg.interval) 652 } 653 "ctpop" => { 654 let [arg] = args else { 655 return Err(MirEvalError::TypeError("likely arg is not provided")); 656 }; 657 let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).count_ones(); 658 destination 659 .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size]) 660 } 661 "cttz" | "cttz_nonzero" => { 662 let [arg] = args else { 663 return Err(MirEvalError::TypeError("likely arg is not provided")); 664 }; 665 let result = u128::from_le_bytes(pad16(arg.get(self)?, false)).trailing_zeros(); 666 destination 667 .write_from_bytes(self, &(result as u128).to_le_bytes()[0..destination.size]) 668 } 669 "const_eval_select" => { 670 let [tuple, const_fn, _] = args else { 671 return Err(MirEvalError::TypeError("const_eval_select args are not provided")); 672 }; 673 let mut args = vec![const_fn.clone()]; 674 let TyKind::Tuple(_, fields) = tuple.ty.kind(Interner) else { 675 return Err(MirEvalError::TypeError("const_eval_select arg[0] is not a tuple")); 676 }; 677 let layout = self.layout(&tuple.ty)?; 678 for (i, field) in fields.iter(Interner).enumerate() { 679 let field = field.assert_ty_ref(Interner).clone(); 680 let offset = layout.fields.offset(i).bytes_usize(); 681 let addr = tuple.interval.addr.offset(offset); 682 args.push(IntervalAndTy::new(addr, field, self, locals)?); 683 } 684 self.exec_fn_trait(&args, destination, locals, span) 685 } 686 _ => not_supported!("unknown intrinsic {name}"), 687 } 688 } 689 exec_atomic_intrinsic( &mut self, name: &str, args: &[IntervalAndTy], generic_args: &Substitution, destination: Interval, locals: &Locals<'_>, _span: MirSpan, ) -> Result<()>690 fn exec_atomic_intrinsic( 691 &mut self, 692 name: &str, 693 args: &[IntervalAndTy], 694 generic_args: &Substitution, 695 destination: Interval, 696 locals: &Locals<'_>, 697 _span: MirSpan, 698 ) -> Result<()> { 699 // We are a single threaded runtime with no UB checking and no optimization, so 700 // we can implement these as normal functions. 701 let Some(ty) = generic_args.as_slice(Interner).get(0).and_then(|x| x.ty(Interner)) else { 702 return Err(MirEvalError::TypeError("atomic intrinsic generic arg is not provided")); 703 }; 704 let Some(arg0) = args.get(0) else { 705 return Err(MirEvalError::TypeError("atomic intrinsic arg0 is not provided")); 706 }; 707 let arg0_addr = Address::from_bytes(arg0.get(self)?)?; 708 let arg0_interval = 709 Interval::new(arg0_addr, self.size_of_sized(ty, locals, "atomic intrinsic type arg")?); 710 if name.starts_with("load_") { 711 return destination.write_from_interval(self, arg0_interval); 712 } 713 let Some(arg1) = args.get(1) else { 714 return Err(MirEvalError::TypeError("atomic intrinsic arg1 is not provided")); 715 }; 716 if name.starts_with("store_") { 717 return arg0_interval.write_from_interval(self, arg1.interval); 718 } 719 if name.starts_with("xchg_") { 720 destination.write_from_interval(self, arg0_interval)?; 721 return arg0_interval.write_from_interval(self, arg1.interval); 722 } 723 if name.starts_with("xadd_") { 724 destination.write_from_interval(self, arg0_interval)?; 725 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 726 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 727 let ans = lhs.wrapping_add(rhs); 728 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 729 } 730 if name.starts_with("xsub_") { 731 destination.write_from_interval(self, arg0_interval)?; 732 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 733 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 734 let ans = lhs.wrapping_sub(rhs); 735 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 736 } 737 if name.starts_with("and_") { 738 destination.write_from_interval(self, arg0_interval)?; 739 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 740 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 741 let ans = lhs & rhs; 742 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 743 } 744 if name.starts_with("or_") { 745 destination.write_from_interval(self, arg0_interval)?; 746 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 747 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 748 let ans = lhs | rhs; 749 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 750 } 751 if name.starts_with("xor_") { 752 destination.write_from_interval(self, arg0_interval)?; 753 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 754 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 755 let ans = lhs ^ rhs; 756 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 757 } 758 if name.starts_with("nand_") { 759 destination.write_from_interval(self, arg0_interval)?; 760 let lhs = u128::from_le_bytes(pad16(arg0_interval.get(self)?, false)); 761 let rhs = u128::from_le_bytes(pad16(arg1.get(self)?, false)); 762 let ans = !(lhs & rhs); 763 return arg0_interval.write_from_bytes(self, &ans.to_le_bytes()[0..destination.size]); 764 } 765 let Some(arg2) = args.get(2) else { 766 return Err(MirEvalError::TypeError("atomic intrinsic arg2 is not provided")); 767 }; 768 if name.starts_with("cxchg_") || name.starts_with("cxchgweak_") { 769 let dest = if arg1.get(self)? == arg0_interval.get(self)? { 770 arg0_interval.write_from_interval(self, arg2.interval)?; 771 (arg1.interval, true) 772 } else { 773 (arg0_interval, false) 774 }; 775 let result_ty = TyKind::Tuple( 776 2, 777 Substitution::from_iter(Interner, [ty.clone(), TyBuilder::bool()]), 778 ) 779 .intern(Interner); 780 let layout = self.layout(&result_ty)?; 781 let result = self.make_by_layout( 782 layout.size.bytes_usize(), 783 &layout, 784 None, 785 [IntervalOrOwned::Borrowed(dest.0), IntervalOrOwned::Owned(vec![u8::from(dest.1)])] 786 .into_iter(), 787 )?; 788 return destination.write_from_bytes(self, &result); 789 } 790 not_supported!("unknown atomic intrinsic {name}"); 791 } 792 } 793