1 //! The memory subsystem. 2 //! 3 //! Generally, we use `Pointer` to denote memory addresses. However, some operations 4 //! have a "size"-like parameter, and they take `Scalar` for the address because 5 //! if the size is 0, then the pointer can also be a (properly aligned, non-null) 6 //! integer. It is crucial that these operations call `check_align` *before* 7 //! short-circuiting the empty case! 8 9 use std::assert_matches::assert_matches; 10 use std::borrow::Cow; 11 use std::collections::VecDeque; 12 use std::fmt; 13 use std::ptr; 14 15 use rustc_ast::Mutability; 16 use rustc_data_structures::fx::{FxHashMap, FxHashSet}; 17 use rustc_middle::mir::display_allocation; 18 use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt}; 19 use rustc_target::abi::{Align, HasDataLayout, Size}; 20 21 use crate::const_eval::CheckAlignment; 22 use crate::fluent_generated as fluent; 23 24 use super::{ 25 alloc_range, AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckInAllocMsg, 26 GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Pointer, PointerArithmetic, Provenance, 27 Scalar, 28 }; 29 30 #[derive(Debug, PartialEq, Copy, Clone)] 31 pub enum MemoryKind<T> { 32 /// Stack memory. Error if deallocated except during a stack pop. 33 Stack, 34 /// Memory allocated by `caller_location` intrinsic. Error if ever deallocated. 35 CallerLocation, 36 /// Additional memory kinds a machine wishes to distinguish from the builtin ones. 37 Machine(T), 38 } 39 40 impl<T: MayLeak> MayLeak for MemoryKind<T> { 41 #[inline] may_leak(self) -> bool42 fn may_leak(self) -> bool { 43 match self { 44 MemoryKind::Stack => false, 45 MemoryKind::CallerLocation => true, 46 MemoryKind::Machine(k) => k.may_leak(), 47 } 48 } 49 } 50 51 impl<T: fmt::Display> fmt::Display for MemoryKind<T> { fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result52 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 53 match self { 54 MemoryKind::Stack => write!(f, "stack variable"), 55 MemoryKind::CallerLocation => write!(f, "caller location"), 56 MemoryKind::Machine(m) => write!(f, "{}", m), 57 } 58 } 59 } 60 61 /// The return value of `get_alloc_info` indicates the "kind" of the allocation. 62 pub enum AllocKind { 63 /// A regular live data allocation. 64 LiveData, 65 /// A function allocation (that fn ptrs point to). 66 Function, 67 /// A (symbolic) vtable allocation. 68 VTable, 69 /// A dead allocation. 70 Dead, 71 } 72 73 /// The value of a function pointer. 74 #[derive(Debug, Copy, Clone)] 75 pub enum FnVal<'tcx, Other> { 76 Instance(Instance<'tcx>), 77 Other(Other), 78 } 79 80 impl<'tcx, Other> FnVal<'tcx, Other> { as_instance(self) -> InterpResult<'tcx, Instance<'tcx>>81 pub fn as_instance(self) -> InterpResult<'tcx, Instance<'tcx>> { 82 match self { 83 FnVal::Instance(instance) => Ok(instance), 84 FnVal::Other(_) => { 85 throw_unsup_format!("'foreign' function pointers are not supported in this context") 86 } 87 } 88 } 89 } 90 91 // `Memory` has to depend on the `Machine` because some of its operations 92 // (e.g., `get`) call a `Machine` hook. 93 pub struct Memory<'mir, 'tcx, M: Machine<'mir, 'tcx>> { 94 /// Allocations local to this instance of the miri engine. The kind 95 /// helps ensure that the same mechanism is used for allocation and 96 /// deallocation. When an allocation is not found here, it is a 97 /// global and looked up in the `tcx` for read access. Some machines may 98 /// have to mutate this map even on a read-only access to a global (because 99 /// they do pointer provenance tracking and the allocations in `tcx` have 100 /// the wrong type), so we let the machine override this type. 101 /// Either way, if the machine allows writing to a global, doing so will 102 /// create a copy of the global allocation here. 103 // FIXME: this should not be public, but interning currently needs access to it 104 pub(super) alloc_map: M::MemoryMap, 105 106 /// Map for "extra" function pointers. 107 extra_fn_ptr_map: FxHashMap<AllocId, M::ExtraFnVal>, 108 109 /// To be able to compare pointers with null, and to check alignment for accesses 110 /// to ZSTs (where pointers may dangle), we keep track of the size even for allocations 111 /// that do not exist any more. 112 // FIXME: this should not be public, but interning currently needs access to it 113 pub(super) dead_alloc_map: FxHashMap<AllocId, (Size, Align)>, 114 } 115 116 /// A reference to some allocation that was already bounds-checked for the given region 117 /// and had the on-access machine hooks run. 118 #[derive(Copy, Clone)] 119 pub struct AllocRef<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> { 120 alloc: &'a Allocation<Prov, Extra, Bytes>, 121 range: AllocRange, 122 tcx: TyCtxt<'tcx>, 123 alloc_id: AllocId, 124 } 125 /// A reference to some allocation that was already bounds-checked for the given region 126 /// and had the on-access machine hooks run. 127 pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> { 128 alloc: &'a mut Allocation<Prov, Extra, Bytes>, 129 range: AllocRange, 130 tcx: TyCtxt<'tcx>, 131 alloc_id: AllocId, 132 } 133 134 impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { new() -> Self135 pub fn new() -> Self { 136 Memory { 137 alloc_map: M::MemoryMap::default(), 138 extra_fn_ptr_map: FxHashMap::default(), 139 dead_alloc_map: FxHashMap::default(), 140 } 141 } 142 143 /// This is used by [priroda](https://github.com/oli-obk/priroda) alloc_map(&self) -> &M::MemoryMap144 pub fn alloc_map(&self) -> &M::MemoryMap { 145 &self.alloc_map 146 } 147 } 148 149 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { 150 /// Call this to turn untagged "global" pointers (obtained via `tcx`) into 151 /// the machine pointer to the allocation. Must never be used 152 /// for any other pointers, nor for TLS statics. 153 /// 154 /// Using the resulting pointer represents a *direct* access to that memory 155 /// (e.g. by directly using a `static`), 156 /// as opposed to access through a pointer that was created by the program. 157 /// 158 /// This function can fail only if `ptr` points to an `extern static`. 159 #[inline] global_base_pointer( &self, ptr: Pointer<AllocId>, ) -> InterpResult<'tcx, Pointer<M::Provenance>>160 pub fn global_base_pointer( 161 &self, 162 ptr: Pointer<AllocId>, 163 ) -> InterpResult<'tcx, Pointer<M::Provenance>> { 164 let alloc_id = ptr.provenance; 165 // We need to handle `extern static`. 166 match self.tcx.try_get_global_alloc(alloc_id) { 167 Some(GlobalAlloc::Static(def_id)) if self.tcx.is_thread_local_static(def_id) => { 168 bug!("global memory cannot point to thread-local static") 169 } 170 Some(GlobalAlloc::Static(def_id)) if self.tcx.is_foreign_item(def_id) => { 171 return M::extern_static_base_pointer(self, def_id); 172 } 173 _ => {} 174 } 175 // And we need to get the provenance. 176 M::adjust_alloc_base_pointer(self, ptr) 177 } 178 create_fn_alloc_ptr( &mut self, fn_val: FnVal<'tcx, M::ExtraFnVal>, ) -> Pointer<M::Provenance>179 pub fn create_fn_alloc_ptr( 180 &mut self, 181 fn_val: FnVal<'tcx, M::ExtraFnVal>, 182 ) -> Pointer<M::Provenance> { 183 let id = match fn_val { 184 FnVal::Instance(instance) => self.tcx.create_fn_alloc(instance), 185 FnVal::Other(extra) => { 186 // FIXME(RalfJung): Should we have a cache here? 187 let id = self.tcx.reserve_alloc_id(); 188 let old = self.memory.extra_fn_ptr_map.insert(id, extra); 189 assert!(old.is_none()); 190 id 191 } 192 }; 193 // Functions are global allocations, so make sure we get the right base pointer. 194 // We know this is not an `extern static` so this cannot fail. 195 self.global_base_pointer(Pointer::from(id)).unwrap() 196 } 197 allocate_ptr( &mut self, size: Size, align: Align, kind: MemoryKind<M::MemoryKind>, ) -> InterpResult<'tcx, Pointer<M::Provenance>>198 pub fn allocate_ptr( 199 &mut self, 200 size: Size, 201 align: Align, 202 kind: MemoryKind<M::MemoryKind>, 203 ) -> InterpResult<'tcx, Pointer<M::Provenance>> { 204 let alloc = if M::PANIC_ON_ALLOC_FAIL { 205 Allocation::uninit(size, align) 206 } else { 207 Allocation::try_uninit(size, align)? 208 }; 209 self.allocate_raw_ptr(alloc, kind) 210 } 211 allocate_bytes_ptr( &mut self, bytes: &[u8], align: Align, kind: MemoryKind<M::MemoryKind>, mutability: Mutability, ) -> InterpResult<'tcx, Pointer<M::Provenance>>212 pub fn allocate_bytes_ptr( 213 &mut self, 214 bytes: &[u8], 215 align: Align, 216 kind: MemoryKind<M::MemoryKind>, 217 mutability: Mutability, 218 ) -> InterpResult<'tcx, Pointer<M::Provenance>> { 219 let alloc = Allocation::from_bytes(bytes, align, mutability); 220 self.allocate_raw_ptr(alloc, kind) 221 } 222 223 /// This can fail only if `alloc` contains provenance. allocate_raw_ptr( &mut self, alloc: Allocation, kind: MemoryKind<M::MemoryKind>, ) -> InterpResult<'tcx, Pointer<M::Provenance>>224 pub fn allocate_raw_ptr( 225 &mut self, 226 alloc: Allocation, 227 kind: MemoryKind<M::MemoryKind>, 228 ) -> InterpResult<'tcx, Pointer<M::Provenance>> { 229 let id = self.tcx.reserve_alloc_id(); 230 debug_assert_ne!( 231 Some(kind), 232 M::GLOBAL_KIND.map(MemoryKind::Machine), 233 "dynamically allocating global memory" 234 ); 235 let alloc = M::adjust_allocation(self, id, Cow::Owned(alloc), Some(kind))?; 236 self.memory.alloc_map.insert(id, (kind, alloc.into_owned())); 237 M::adjust_alloc_base_pointer(self, Pointer::from(id)) 238 } 239 reallocate_ptr( &mut self, ptr: Pointer<Option<M::Provenance>>, old_size_and_align: Option<(Size, Align)>, new_size: Size, new_align: Align, kind: MemoryKind<M::MemoryKind>, ) -> InterpResult<'tcx, Pointer<M::Provenance>>240 pub fn reallocate_ptr( 241 &mut self, 242 ptr: Pointer<Option<M::Provenance>>, 243 old_size_and_align: Option<(Size, Align)>, 244 new_size: Size, 245 new_align: Align, 246 kind: MemoryKind<M::MemoryKind>, 247 ) -> InterpResult<'tcx, Pointer<M::Provenance>> { 248 let (alloc_id, offset, _prov) = self.ptr_get_alloc_id(ptr)?; 249 if offset.bytes() != 0 { 250 throw_ub_custom!( 251 fluent::const_eval_realloc_or_alloc_with_offset, 252 ptr = format!("{ptr:?}"), 253 kind = "realloc" 254 ); 255 } 256 257 // For simplicities' sake, we implement reallocate as "alloc, copy, dealloc". 258 // This happens so rarely, the perf advantage is outweighed by the maintenance cost. 259 let new_ptr = self.allocate_ptr(new_size, new_align, kind)?; 260 let old_size = match old_size_and_align { 261 Some((size, _align)) => size, 262 None => self.get_alloc_raw(alloc_id)?.size(), 263 }; 264 // This will also call the access hooks. 265 self.mem_copy( 266 ptr, 267 Align::ONE, 268 new_ptr.into(), 269 Align::ONE, 270 old_size.min(new_size), 271 /*nonoverlapping*/ true, 272 )?; 273 self.deallocate_ptr(ptr, old_size_and_align, kind)?; 274 275 Ok(new_ptr) 276 } 277 278 #[instrument(skip(self), level = "debug")] deallocate_ptr( &mut self, ptr: Pointer<Option<M::Provenance>>, old_size_and_align: Option<(Size, Align)>, kind: MemoryKind<M::MemoryKind>, ) -> InterpResult<'tcx>279 pub fn deallocate_ptr( 280 &mut self, 281 ptr: Pointer<Option<M::Provenance>>, 282 old_size_and_align: Option<(Size, Align)>, 283 kind: MemoryKind<M::MemoryKind>, 284 ) -> InterpResult<'tcx> { 285 let (alloc_id, offset, prov) = self.ptr_get_alloc_id(ptr)?; 286 trace!("deallocating: {alloc_id:?}"); 287 288 if offset.bytes() != 0 { 289 throw_ub_custom!( 290 fluent::const_eval_realloc_or_alloc_with_offset, 291 ptr = format!("{ptr:?}"), 292 kind = "dealloc", 293 ); 294 } 295 296 let Some((alloc_kind, mut alloc)) = self.memory.alloc_map.remove(&alloc_id) else { 297 // Deallocating global memory -- always an error 298 return Err(match self.tcx.try_get_global_alloc(alloc_id) { 299 Some(GlobalAlloc::Function(..)) => { 300 err_ub_custom!( 301 fluent::const_eval_invalid_dealloc, 302 alloc_id = alloc_id, 303 kind = "fn", 304 ) 305 } 306 Some(GlobalAlloc::VTable(..)) => { 307 err_ub_custom!( 308 fluent::const_eval_invalid_dealloc, 309 alloc_id = alloc_id, 310 kind = "vtable", 311 ) 312 } 313 Some(GlobalAlloc::Static(..) | GlobalAlloc::Memory(..)) => { 314 err_ub_custom!( 315 fluent::const_eval_invalid_dealloc, 316 alloc_id = alloc_id, 317 kind = "static_mem" 318 ) 319 } 320 None => err_ub!(PointerUseAfterFree(alloc_id)), 321 } 322 .into()); 323 }; 324 325 if alloc.mutability.is_not() { 326 throw_ub_custom!(fluent::const_eval_dealloc_immutable, alloc = alloc_id,); 327 } 328 if alloc_kind != kind { 329 throw_ub_custom!( 330 fluent::const_eval_dealloc_kind_mismatch, 331 alloc = alloc_id, 332 alloc_kind = format!("{alloc_kind}"), 333 kind = format!("{kind}"), 334 ); 335 } 336 if let Some((size, align)) = old_size_and_align { 337 if size != alloc.size() || align != alloc.align { 338 throw_ub_custom!( 339 fluent::const_eval_dealloc_incorrect_layout, 340 alloc = alloc_id, 341 size = alloc.size().bytes(), 342 align = alloc.align.bytes(), 343 size_found = size.bytes(), 344 align_found = align.bytes(), 345 ) 346 } 347 } 348 349 // Let the machine take some extra action 350 let size = alloc.size(); 351 M::before_memory_deallocation( 352 *self.tcx, 353 &mut self.machine, 354 &mut alloc.extra, 355 (alloc_id, prov), 356 alloc_range(Size::ZERO, size), 357 )?; 358 359 // Don't forget to remember size and align of this now-dead allocation 360 let old = self.memory.dead_alloc_map.insert(alloc_id, (size, alloc.align)); 361 if old.is_some() { 362 bug!("Nothing can be deallocated twice"); 363 } 364 365 Ok(()) 366 } 367 368 /// Internal helper function to determine the allocation and offset of a pointer (if any). 369 #[inline(always)] get_ptr_access( &self, ptr: Pointer<Option<M::Provenance>>, size: Size, align: Align, ) -> InterpResult<'tcx, Option<(AllocId, Size, M::ProvenanceExtra)>>370 fn get_ptr_access( 371 &self, 372 ptr: Pointer<Option<M::Provenance>>, 373 size: Size, 374 align: Align, 375 ) -> InterpResult<'tcx, Option<(AllocId, Size, M::ProvenanceExtra)>> { 376 self.check_and_deref_ptr( 377 ptr, 378 size, 379 align, 380 M::enforce_alignment(self), 381 CheckInAllocMsg::MemoryAccessTest, 382 |alloc_id, offset, prov| { 383 let (size, align) = self.get_live_alloc_size_and_align(alloc_id)?; 384 Ok((size, align, (alloc_id, offset, prov))) 385 }, 386 ) 387 } 388 389 /// Check if the given pointer points to live memory of given `size` and `align` 390 /// (ignoring `M::enforce_alignment`). The caller can control the error message for the 391 /// out-of-bounds case. 392 #[inline(always)] check_ptr_access_align( &self, ptr: Pointer<Option<M::Provenance>>, size: Size, align: Align, msg: CheckInAllocMsg, ) -> InterpResult<'tcx>393 pub fn check_ptr_access_align( 394 &self, 395 ptr: Pointer<Option<M::Provenance>>, 396 size: Size, 397 align: Align, 398 msg: CheckInAllocMsg, 399 ) -> InterpResult<'tcx> { 400 self.check_and_deref_ptr( 401 ptr, 402 size, 403 align, 404 CheckAlignment::Error, 405 msg, 406 |alloc_id, _, _| { 407 let (size, align) = self.get_live_alloc_size_and_align(alloc_id)?; 408 Ok((size, align, ())) 409 }, 410 )?; 411 Ok(()) 412 } 413 414 /// Low-level helper function to check if a ptr is in-bounds and potentially return a reference 415 /// to the allocation it points to. Supports both shared and mutable references, as the actual 416 /// checking is offloaded to a helper closure. `align` defines whether and which alignment check 417 /// is done. Returns `None` for size 0, and otherwise `Some` of what `alloc_size` returned. check_and_deref_ptr<T>( &self, ptr: Pointer<Option<M::Provenance>>, size: Size, align: Align, check: CheckAlignment, msg: CheckInAllocMsg, alloc_size: impl FnOnce( AllocId, Size, M::ProvenanceExtra, ) -> InterpResult<'tcx, (Size, Align, T)>, ) -> InterpResult<'tcx, Option<T>>418 fn check_and_deref_ptr<T>( 419 &self, 420 ptr: Pointer<Option<M::Provenance>>, 421 size: Size, 422 align: Align, 423 check: CheckAlignment, 424 msg: CheckInAllocMsg, 425 alloc_size: impl FnOnce( 426 AllocId, 427 Size, 428 M::ProvenanceExtra, 429 ) -> InterpResult<'tcx, (Size, Align, T)>, 430 ) -> InterpResult<'tcx, Option<T>> { 431 Ok(match self.ptr_try_get_alloc_id(ptr) { 432 Err(addr) => { 433 // We couldn't get a proper allocation. This is only okay if the access size is 0, 434 // and the address is not null. 435 if size.bytes() > 0 || addr == 0 { 436 throw_ub!(DanglingIntPointer(addr, msg)); 437 } 438 // Must be aligned. 439 if check.should_check() { 440 self.check_offset_align(addr, align, check)?; 441 } 442 None 443 } 444 Ok((alloc_id, offset, prov)) => { 445 let (alloc_size, alloc_align, ret_val) = alloc_size(alloc_id, offset, prov)?; 446 // Test bounds. This also ensures non-null. 447 // It is sufficient to check this for the end pointer. Also check for overflow! 448 if offset.checked_add(size, &self.tcx).map_or(true, |end| end > alloc_size) { 449 throw_ub!(PointerOutOfBounds { 450 alloc_id, 451 alloc_size, 452 ptr_offset: self.target_usize_to_isize(offset.bytes()), 453 ptr_size: size, 454 msg, 455 }) 456 } 457 // Ensure we never consider the null pointer dereferenceable. 458 if M::Provenance::OFFSET_IS_ADDR { 459 assert_ne!(ptr.addr(), Size::ZERO); 460 } 461 // Test align. Check this last; if both bounds and alignment are violated 462 // we want the error to be about the bounds. 463 if check.should_check() { 464 if M::use_addr_for_alignment_check(self) { 465 // `use_addr_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true. 466 self.check_offset_align(ptr.addr().bytes(), align, check)?; 467 } else { 468 // Check allocation alignment and offset alignment. 469 if alloc_align.bytes() < align.bytes() { 470 M::alignment_check_failed(self, alloc_align, align, check)?; 471 } 472 self.check_offset_align(offset.bytes(), align, check)?; 473 } 474 } 475 476 // We can still be zero-sized in this branch, in which case we have to 477 // return `None`. 478 if size.bytes() == 0 { None } else { Some(ret_val) } 479 } 480 }) 481 } 482 check_offset_align( &self, offset: u64, align: Align, check: CheckAlignment, ) -> InterpResult<'tcx>483 fn check_offset_align( 484 &self, 485 offset: u64, 486 align: Align, 487 check: CheckAlignment, 488 ) -> InterpResult<'tcx> { 489 if offset % align.bytes() == 0 { 490 Ok(()) 491 } else { 492 // The biggest power of two through which `offset` is divisible. 493 let offset_pow2 = 1 << offset.trailing_zeros(); 494 M::alignment_check_failed(self, Align::from_bytes(offset_pow2).unwrap(), align, check) 495 } 496 } 497 } 498 499 /// Allocation accessors 500 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { 501 /// Helper function to obtain a global (tcx) allocation. 502 /// This attempts to return a reference to an existing allocation if 503 /// one can be found in `tcx`. That, however, is only possible if `tcx` and 504 /// this machine use the same pointer provenance, so it is indirected through 505 /// `M::adjust_allocation`. get_global_alloc( &self, id: AllocId, is_write: bool, ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::Provenance, M::AllocExtra, M::Bytes>>>506 fn get_global_alloc( 507 &self, 508 id: AllocId, 509 is_write: bool, 510 ) -> InterpResult<'tcx, Cow<'tcx, Allocation<M::Provenance, M::AllocExtra, M::Bytes>>> { 511 let (alloc, def_id) = match self.tcx.try_get_global_alloc(id) { 512 Some(GlobalAlloc::Memory(mem)) => { 513 // Memory of a constant or promoted or anonymous memory referenced by a static. 514 (mem, None) 515 } 516 Some(GlobalAlloc::Function(..)) => throw_ub!(DerefFunctionPointer(id)), 517 Some(GlobalAlloc::VTable(..)) => throw_ub!(DerefVTablePointer(id)), 518 None => throw_ub!(PointerUseAfterFree(id)), 519 Some(GlobalAlloc::Static(def_id)) => { 520 assert!(self.tcx.is_static(def_id)); 521 assert!(!self.tcx.is_thread_local_static(def_id)); 522 // Notice that every static has two `AllocId` that will resolve to the same 523 // thing here: one maps to `GlobalAlloc::Static`, this is the "lazy" ID, 524 // and the other one is maps to `GlobalAlloc::Memory`, this is returned by 525 // `eval_static_initializer` and it is the "resolved" ID. 526 // The resolved ID is never used by the interpreted program, it is hidden. 527 // This is relied upon for soundness of const-patterns; a pointer to the resolved 528 // ID would "sidestep" the checks that make sure consts do not point to statics! 529 // The `GlobalAlloc::Memory` branch here is still reachable though; when a static 530 // contains a reference to memory that was created during its evaluation (i.e., not 531 // to another static), those inner references only exist in "resolved" form. 532 if self.tcx.is_foreign_item(def_id) { 533 // This is unreachable in Miri, but can happen in CTFE where we actually *do* support 534 // referencing arbitrary (declared) extern statics. 535 throw_unsup!(ReadExternStatic(def_id)); 536 } 537 538 // We don't give a span -- statics don't need that, they cannot be generic or associated. 539 let val = self.ctfe_query(None, |tcx| tcx.eval_static_initializer(def_id))?; 540 (val, Some(def_id)) 541 } 542 }; 543 M::before_access_global(*self.tcx, &self.machine, id, alloc, def_id, is_write)?; 544 // We got tcx memory. Let the machine initialize its "extra" stuff. 545 M::adjust_allocation( 546 self, 547 id, // always use the ID we got as input, not the "hidden" one. 548 Cow::Borrowed(alloc.inner()), 549 M::GLOBAL_KIND.map(MemoryKind::Machine), 550 ) 551 } 552 553 /// Get the base address for the bytes in an `Allocation` specified by the 554 /// `AllocID` passed in; error if no such allocation exists. 555 /// 556 /// It is up to the caller to take sufficient care when using this address: 557 /// there could be provenance or uninit memory in there, and other memory 558 /// accesses could invalidate the exposed pointer. alloc_base_addr(&self, id: AllocId) -> InterpResult<'tcx, *const u8>559 pub fn alloc_base_addr(&self, id: AllocId) -> InterpResult<'tcx, *const u8> { 560 let alloc = self.get_alloc_raw(id)?; 561 Ok(alloc.base_addr()) 562 } 563 564 /// Gives raw access to the `Allocation`, without bounds or alignment checks. 565 /// The caller is responsible for calling the access hooks! 566 /// 567 /// You almost certainly want to use `get_ptr_alloc`/`get_ptr_alloc_mut` instead. get_alloc_raw( &self, id: AllocId, ) -> InterpResult<'tcx, &Allocation<M::Provenance, M::AllocExtra, M::Bytes>>568 fn get_alloc_raw( 569 &self, 570 id: AllocId, 571 ) -> InterpResult<'tcx, &Allocation<M::Provenance, M::AllocExtra, M::Bytes>> { 572 // The error type of the inner closure here is somewhat funny. We have two 573 // ways of "erroring": An actual error, or because we got a reference from 574 // `get_global_alloc` that we can actually use directly without inserting anything anywhere. 575 // So the error type is `InterpResult<'tcx, &Allocation<M::Provenance>>`. 576 let a = self.memory.alloc_map.get_or(id, || { 577 let alloc = self.get_global_alloc(id, /*is_write*/ false).map_err(Err)?; 578 match alloc { 579 Cow::Borrowed(alloc) => { 580 // We got a ref, cheaply return that as an "error" so that the 581 // map does not get mutated. 582 Err(Ok(alloc)) 583 } 584 Cow::Owned(alloc) => { 585 // Need to put it into the map and return a ref to that 586 let kind = M::GLOBAL_KIND.expect( 587 "I got a global allocation that I have to copy but the machine does \ 588 not expect that to happen", 589 ); 590 Ok((MemoryKind::Machine(kind), alloc)) 591 } 592 } 593 }); 594 // Now unpack that funny error type 595 match a { 596 Ok(a) => Ok(&a.1), 597 Err(a) => a, 598 } 599 } 600 601 /// "Safe" (bounds and align-checked) allocation access. get_ptr_alloc<'a>( &'a self, ptr: Pointer<Option<M::Provenance>>, size: Size, align: Align, ) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>602 pub fn get_ptr_alloc<'a>( 603 &'a self, 604 ptr: Pointer<Option<M::Provenance>>, 605 size: Size, 606 align: Align, 607 ) -> InterpResult<'tcx, Option<AllocRef<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>> 608 { 609 let ptr_and_alloc = self.check_and_deref_ptr( 610 ptr, 611 size, 612 align, 613 M::enforce_alignment(self), 614 CheckInAllocMsg::MemoryAccessTest, 615 |alloc_id, offset, prov| { 616 let alloc = self.get_alloc_raw(alloc_id)?; 617 Ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc))) 618 }, 619 )?; 620 if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc { 621 let range = alloc_range(offset, size); 622 M::before_memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?; 623 Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id })) 624 } else { 625 // Even in this branch we have to be sure that we actually access the allocation, in 626 // order to ensure that `static FOO: Type = FOO;` causes a cycle error instead of 627 // magically pulling *any* ZST value from the ether. However, the `get_raw` above is 628 // always called when `ptr` has an `AllocId`. 629 Ok(None) 630 } 631 } 632 633 /// Return the `extra` field of the given allocation. get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra>634 pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> { 635 Ok(&self.get_alloc_raw(id)?.extra) 636 } 637 638 /// Return the `mutability` field of the given allocation. get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability>639 pub fn get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability> { 640 Ok(self.get_alloc_raw(id)?.mutability) 641 } 642 643 /// Gives raw mutable access to the `Allocation`, without bounds or alignment checks. 644 /// The caller is responsible for calling the access hooks! 645 /// 646 /// Also returns a ptr to `self.extra` so that the caller can use it in parallel with the 647 /// allocation. get_alloc_raw_mut( &mut self, id: AllocId, ) -> InterpResult<'tcx, (&mut Allocation<M::Provenance, M::AllocExtra, M::Bytes>, &mut M)>648 fn get_alloc_raw_mut( 649 &mut self, 650 id: AllocId, 651 ) -> InterpResult<'tcx, (&mut Allocation<M::Provenance, M::AllocExtra, M::Bytes>, &mut M)> { 652 // We have "NLL problem case #3" here, which cannot be worked around without loss of 653 // efficiency even for the common case where the key is in the map. 654 // <https://rust-lang.github.io/rfcs/2094-nll.html#problem-case-3-conditional-control-flow-across-functions> 655 // (Cannot use `get_mut_or` since `get_global_alloc` needs `&self`.) 656 if self.memory.alloc_map.get_mut(id).is_none() { 657 // Slow path. 658 // Allocation not found locally, go look global. 659 let alloc = self.get_global_alloc(id, /*is_write*/ true)?; 660 let kind = M::GLOBAL_KIND.expect( 661 "I got a global allocation that I have to copy but the machine does \ 662 not expect that to happen", 663 ); 664 self.memory.alloc_map.insert(id, (MemoryKind::Machine(kind), alloc.into_owned())); 665 } 666 667 let (_kind, alloc) = self.memory.alloc_map.get_mut(id).unwrap(); 668 if alloc.mutability.is_not() { 669 throw_ub!(WriteToReadOnly(id)) 670 } 671 Ok((alloc, &mut self.machine)) 672 } 673 674 /// "Safe" (bounds and align-checked) allocation access. get_ptr_alloc_mut<'a>( &'a mut self, ptr: Pointer<Option<M::Provenance>>, size: Size, align: Align, ) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>675 pub fn get_ptr_alloc_mut<'a>( 676 &'a mut self, 677 ptr: Pointer<Option<M::Provenance>>, 678 size: Size, 679 align: Align, 680 ) -> InterpResult<'tcx, Option<AllocRefMut<'a, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>> 681 { 682 let parts = self.get_ptr_access(ptr, size, align)?; 683 if let Some((alloc_id, offset, prov)) = parts { 684 let tcx = *self.tcx; 685 // FIXME: can we somehow avoid looking up the allocation twice here? 686 // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`. 687 let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?; 688 let range = alloc_range(offset, size); 689 M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?; 690 Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id })) 691 } else { 692 Ok(None) 693 } 694 } 695 696 /// Return the `extra` field of the given allocation. get_alloc_extra_mut<'a>( &'a mut self, id: AllocId, ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)>697 pub fn get_alloc_extra_mut<'a>( 698 &'a mut self, 699 id: AllocId, 700 ) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)> { 701 let (alloc, machine) = self.get_alloc_raw_mut(id)?; 702 Ok((&mut alloc.extra, machine)) 703 } 704 705 /// Obtain the size and alignment of an allocation, even if that allocation has 706 /// been deallocated. get_alloc_info(&self, id: AllocId) -> (Size, Align, AllocKind)707 pub fn get_alloc_info(&self, id: AllocId) -> (Size, Align, AllocKind) { 708 // # Regular allocations 709 // Don't use `self.get_raw` here as that will 710 // a) cause cycles in case `id` refers to a static 711 // b) duplicate a global's allocation in miri 712 if let Some((_, alloc)) = self.memory.alloc_map.get(id) { 713 return (alloc.size(), alloc.align, AllocKind::LiveData); 714 } 715 716 // # Function pointers 717 // (both global from `alloc_map` and local from `extra_fn_ptr_map`) 718 if self.get_fn_alloc(id).is_some() { 719 return (Size::ZERO, Align::ONE, AllocKind::Function); 720 } 721 722 // # Statics 723 // Can't do this in the match argument, we may get cycle errors since the lock would 724 // be held throughout the match. 725 match self.tcx.try_get_global_alloc(id) { 726 Some(GlobalAlloc::Static(def_id)) => { 727 assert!(self.tcx.is_static(def_id)); 728 assert!(!self.tcx.is_thread_local_static(def_id)); 729 // Use size and align of the type. 730 let ty = self 731 .tcx 732 .type_of(def_id) 733 .no_bound_vars() 734 .expect("statics should not have generic parameters"); 735 let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap(); 736 assert!(layout.is_sized()); 737 (layout.size, layout.align.abi, AllocKind::LiveData) 738 } 739 Some(GlobalAlloc::Memory(alloc)) => { 740 // Need to duplicate the logic here, because the global allocations have 741 // different associated types than the interpreter-local ones. 742 let alloc = alloc.inner(); 743 (alloc.size(), alloc.align, AllocKind::LiveData) 744 } 745 Some(GlobalAlloc::Function(_)) => bug!("We already checked function pointers above"), 746 Some(GlobalAlloc::VTable(..)) => { 747 // No data to be accessed here. But vtables are pointer-aligned. 748 return (Size::ZERO, self.tcx.data_layout.pointer_align.abi, AllocKind::VTable); 749 } 750 // The rest must be dead. 751 None => { 752 // Deallocated pointers are allowed, we should be able to find 753 // them in the map. 754 let (size, align) = *self 755 .memory 756 .dead_alloc_map 757 .get(&id) 758 .expect("deallocated pointers should all be recorded in `dead_alloc_map`"); 759 (size, align, AllocKind::Dead) 760 } 761 } 762 } 763 764 /// Obtain the size and alignment of a live allocation. get_live_alloc_size_and_align(&self, id: AllocId) -> InterpResult<'tcx, (Size, Align)>765 pub fn get_live_alloc_size_and_align(&self, id: AllocId) -> InterpResult<'tcx, (Size, Align)> { 766 let (size, align, kind) = self.get_alloc_info(id); 767 if matches!(kind, AllocKind::Dead) { 768 throw_ub!(PointerUseAfterFree(id)) 769 } 770 Ok((size, align)) 771 } 772 get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>>773 fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> { 774 if let Some(extra) = self.memory.extra_fn_ptr_map.get(&id) { 775 Some(FnVal::Other(*extra)) 776 } else { 777 match self.tcx.try_get_global_alloc(id) { 778 Some(GlobalAlloc::Function(instance)) => Some(FnVal::Instance(instance)), 779 _ => None, 780 } 781 } 782 } 783 get_ptr_fn( &self, ptr: Pointer<Option<M::Provenance>>, ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>>784 pub fn get_ptr_fn( 785 &self, 786 ptr: Pointer<Option<M::Provenance>>, 787 ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> { 788 trace!("get_ptr_fn({:?})", ptr); 789 let (alloc_id, offset, _prov) = self.ptr_get_alloc_id(ptr)?; 790 if offset.bytes() != 0 { 791 throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))) 792 } 793 self.get_fn_alloc(alloc_id) 794 .ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into()) 795 } 796 get_ptr_vtable( &self, ptr: Pointer<Option<M::Provenance>>, ) -> InterpResult<'tcx, (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>)>797 pub fn get_ptr_vtable( 798 &self, 799 ptr: Pointer<Option<M::Provenance>>, 800 ) -> InterpResult<'tcx, (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>)> { 801 trace!("get_ptr_vtable({:?})", ptr); 802 let (alloc_id, offset, _tag) = self.ptr_get_alloc_id(ptr)?; 803 if offset.bytes() != 0 { 804 throw_ub!(InvalidVTablePointer(Pointer::new(alloc_id, offset))) 805 } 806 match self.tcx.try_get_global_alloc(alloc_id) { 807 Some(GlobalAlloc::VTable(ty, trait_ref)) => Ok((ty, trait_ref)), 808 _ => throw_ub!(InvalidVTablePointer(Pointer::new(alloc_id, offset))), 809 } 810 } 811 alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx>812 pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> { 813 self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not; 814 Ok(()) 815 } 816 817 /// Create a lazy debug printer that prints the given allocation and all allocations it points 818 /// to, recursively. 819 #[must_use] dump_alloc<'a>(&'a self, id: AllocId) -> DumpAllocs<'a, 'mir, 'tcx, M>820 pub fn dump_alloc<'a>(&'a self, id: AllocId) -> DumpAllocs<'a, 'mir, 'tcx, M> { 821 self.dump_allocs(vec![id]) 822 } 823 824 /// Create a lazy debug printer for a list of allocations and all allocations they point to, 825 /// recursively. 826 #[must_use] dump_allocs<'a>(&'a self, mut allocs: Vec<AllocId>) -> DumpAllocs<'a, 'mir, 'tcx, M>827 pub fn dump_allocs<'a>(&'a self, mut allocs: Vec<AllocId>) -> DumpAllocs<'a, 'mir, 'tcx, M> { 828 allocs.sort(); 829 allocs.dedup(); 830 DumpAllocs { ecx: self, allocs } 831 } 832 833 /// Find leaked allocations. Allocations reachable from `static_roots` or a `Global` allocation 834 /// are not considered leaked, as well as leaks whose kind's `may_leak()` returns true. find_leaked_allocations( &self, static_roots: &[AllocId], ) -> Vec<(AllocId, MemoryKind<M::MemoryKind>, Allocation<M::Provenance, M::AllocExtra, M::Bytes>)>835 pub fn find_leaked_allocations( 836 &self, 837 static_roots: &[AllocId], 838 ) -> Vec<(AllocId, MemoryKind<M::MemoryKind>, Allocation<M::Provenance, M::AllocExtra, M::Bytes>)> 839 { 840 // Collect the set of allocations that are *reachable* from `Global` allocations. 841 let reachable = { 842 let mut reachable = FxHashSet::default(); 843 let global_kind = M::GLOBAL_KIND.map(MemoryKind::Machine); 844 let mut todo: Vec<_> = 845 self.memory.alloc_map.filter_map_collect(move |&id, &(kind, _)| { 846 if Some(kind) == global_kind { Some(id) } else { None } 847 }); 848 todo.extend(static_roots); 849 while let Some(id) = todo.pop() { 850 if reachable.insert(id) { 851 // This is a new allocation, add the allocation it points to `todo`. 852 if let Some((_, alloc)) = self.memory.alloc_map.get(id) { 853 todo.extend( 854 alloc.provenance().provenances().filter_map(|prov| prov.get_alloc_id()), 855 ); 856 } 857 } 858 } 859 reachable 860 }; 861 862 // All allocations that are *not* `reachable` and *not* `may_leak` are considered leaking. 863 self.memory.alloc_map.filter_map_collect(|id, (kind, alloc)| { 864 if kind.may_leak() || reachable.contains(id) { 865 None 866 } else { 867 Some((*id, *kind, alloc.clone())) 868 } 869 }) 870 } 871 } 872 873 #[doc(hidden)] 874 /// There's no way to use this directly, it's just a helper struct for the `dump_alloc(s)` methods. 875 pub struct DumpAllocs<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> { 876 ecx: &'a InterpCx<'mir, 'tcx, M>, 877 allocs: Vec<AllocId>, 878 } 879 880 impl<'a, 'mir, 'tcx, M: Machine<'mir, 'tcx>> std::fmt::Debug for DumpAllocs<'a, 'mir, 'tcx, M> { fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result881 fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 882 // Cannot be a closure because it is generic in `Prov`, `Extra`. 883 fn write_allocation_track_relocs<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>( 884 fmt: &mut std::fmt::Formatter<'_>, 885 tcx: TyCtxt<'tcx>, 886 allocs_to_print: &mut VecDeque<AllocId>, 887 alloc: &Allocation<Prov, Extra, Bytes>, 888 ) -> std::fmt::Result { 889 for alloc_id in alloc.provenance().provenances().filter_map(|prov| prov.get_alloc_id()) 890 { 891 allocs_to_print.push_back(alloc_id); 892 } 893 write!(fmt, "{}", display_allocation(tcx, alloc)) 894 } 895 896 let mut allocs_to_print: VecDeque<_> = self.allocs.iter().copied().collect(); 897 // `allocs_printed` contains all allocations that we have already printed. 898 let mut allocs_printed = FxHashSet::default(); 899 900 while let Some(id) = allocs_to_print.pop_front() { 901 if !allocs_printed.insert(id) { 902 // Already printed, so skip this. 903 continue; 904 } 905 906 write!(fmt, "{id:?}")?; 907 match self.ecx.memory.alloc_map.get(id) { 908 Some((kind, alloc)) => { 909 // normal alloc 910 write!(fmt, " ({}, ", kind)?; 911 write_allocation_track_relocs( 912 &mut *fmt, 913 *self.ecx.tcx, 914 &mut allocs_to_print, 915 alloc, 916 )?; 917 } 918 None => { 919 // global alloc 920 match self.ecx.tcx.try_get_global_alloc(id) { 921 Some(GlobalAlloc::Memory(alloc)) => { 922 write!(fmt, " (unchanged global, ")?; 923 write_allocation_track_relocs( 924 &mut *fmt, 925 *self.ecx.tcx, 926 &mut allocs_to_print, 927 alloc.inner(), 928 )?; 929 } 930 Some(GlobalAlloc::Function(func)) => { 931 write!(fmt, " (fn: {func})")?; 932 } 933 Some(GlobalAlloc::VTable(ty, Some(trait_ref))) => { 934 write!(fmt, " (vtable: impl {trait_ref} for {ty})")?; 935 } 936 Some(GlobalAlloc::VTable(ty, None)) => { 937 write!(fmt, " (vtable: impl <auto trait> for {ty})")?; 938 } 939 Some(GlobalAlloc::Static(did)) => { 940 write!(fmt, " (static: {})", self.ecx.tcx.def_path_str(did))?; 941 } 942 None => { 943 write!(fmt, " (deallocated)")?; 944 } 945 } 946 } 947 } 948 writeln!(fmt)?; 949 } 950 Ok(()) 951 } 952 } 953 954 /// Reading and writing. 955 impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes> 956 AllocRefMut<'a, 'tcx, Prov, Extra, Bytes> 957 { 958 /// `range` is relative to this allocation reference, not the base of the allocation. write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx>959 pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> { 960 let range = self.range.subrange(range); 961 debug!("write_scalar at {:?}{range:?}: {val:?}", self.alloc_id); 962 Ok(self 963 .alloc 964 .write_scalar(&self.tcx, range, val) 965 .map_err(|e| e.to_interp_error(self.alloc_id))?) 966 } 967 968 /// `offset` is relative to this allocation reference, not the base of the allocation. write_ptr_sized(&mut self, offset: Size, val: Scalar<Prov>) -> InterpResult<'tcx>969 pub fn write_ptr_sized(&mut self, offset: Size, val: Scalar<Prov>) -> InterpResult<'tcx> { 970 self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size), val) 971 } 972 973 /// Mark the entire referenced range as uninitialized write_uninit(&mut self) -> InterpResult<'tcx>974 pub fn write_uninit(&mut self) -> InterpResult<'tcx> { 975 Ok(self 976 .alloc 977 .write_uninit(&self.tcx, self.range) 978 .map_err(|e| e.to_interp_error(self.alloc_id))?) 979 } 980 } 981 982 impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Prov, Extra, Bytes> { 983 /// `range` is relative to this allocation reference, not the base of the allocation. read_scalar( &self, range: AllocRange, read_provenance: bool, ) -> InterpResult<'tcx, Scalar<Prov>>984 pub fn read_scalar( 985 &self, 986 range: AllocRange, 987 read_provenance: bool, 988 ) -> InterpResult<'tcx, Scalar<Prov>> { 989 let range = self.range.subrange(range); 990 let res = self 991 .alloc 992 .read_scalar(&self.tcx, range, read_provenance) 993 .map_err(|e| e.to_interp_error(self.alloc_id))?; 994 debug!("read_scalar at {:?}{range:?}: {res:?}", self.alloc_id); 995 Ok(res) 996 } 997 998 /// `range` is relative to this allocation reference, not the base of the allocation. read_integer(&self, range: AllocRange) -> InterpResult<'tcx, Scalar<Prov>>999 pub fn read_integer(&self, range: AllocRange) -> InterpResult<'tcx, Scalar<Prov>> { 1000 self.read_scalar(range, /*read_provenance*/ false) 1001 } 1002 1003 /// `offset` is relative to this allocation reference, not the base of the allocation. read_pointer(&self, offset: Size) -> InterpResult<'tcx, Scalar<Prov>>1004 pub fn read_pointer(&self, offset: Size) -> InterpResult<'tcx, Scalar<Prov>> { 1005 self.read_scalar( 1006 alloc_range(offset, self.tcx.data_layout().pointer_size), 1007 /*read_provenance*/ true, 1008 ) 1009 } 1010 1011 /// `range` is relative to this allocation reference, not the base of the allocation. get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]>1012 pub fn get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]> { 1013 Ok(self 1014 .alloc 1015 .get_bytes_strip_provenance(&self.tcx, self.range) 1016 .map_err(|e| e.to_interp_error(self.alloc_id))?) 1017 } 1018 1019 /// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`. has_provenance(&self) -> bool1020 pub(crate) fn has_provenance(&self) -> bool { 1021 !self.alloc.provenance().range_empty(self.range, &self.tcx) 1022 } 1023 } 1024 1025 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { 1026 /// Reads the given number of bytes from memory, and strips their provenance if possible. 1027 /// Returns them as a slice. 1028 /// 1029 /// Performs appropriate bounds checks. read_bytes_ptr_strip_provenance( &self, ptr: Pointer<Option<M::Provenance>>, size: Size, ) -> InterpResult<'tcx, &[u8]>1030 pub fn read_bytes_ptr_strip_provenance( 1031 &self, 1032 ptr: Pointer<Option<M::Provenance>>, 1033 size: Size, 1034 ) -> InterpResult<'tcx, &[u8]> { 1035 let Some(alloc_ref) = self.get_ptr_alloc(ptr, size, Align::ONE)? else { 1036 // zero-sized access 1037 return Ok(&[]); 1038 }; 1039 // Side-step AllocRef and directly access the underlying bytes more efficiently. 1040 // (We are staying inside the bounds here so all is good.) 1041 Ok(alloc_ref 1042 .alloc 1043 .get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range) 1044 .map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?) 1045 } 1046 1047 /// Writes the given stream of bytes into memory. 1048 /// 1049 /// Performs appropriate bounds checks. write_bytes_ptr( &mut self, ptr: Pointer<Option<M::Provenance>>, src: impl IntoIterator<Item = u8>, ) -> InterpResult<'tcx>1050 pub fn write_bytes_ptr( 1051 &mut self, 1052 ptr: Pointer<Option<M::Provenance>>, 1053 src: impl IntoIterator<Item = u8>, 1054 ) -> InterpResult<'tcx> { 1055 let mut src = src.into_iter(); 1056 let (lower, upper) = src.size_hint(); 1057 let len = upper.expect("can only write bounded iterators"); 1058 assert_eq!(lower, len, "can only write iterators with a precise length"); 1059 1060 let size = Size::from_bytes(len); 1061 let Some(alloc_ref) = self.get_ptr_alloc_mut(ptr, size, Align::ONE)? else { 1062 // zero-sized access 1063 assert_matches!( 1064 src.next(), 1065 None, 1066 "iterator said it was empty but returned an element" 1067 ); 1068 return Ok(()); 1069 }; 1070 1071 // Side-step AllocRef and directly access the underlying bytes more efficiently. 1072 // (We are staying inside the bounds here so all is good.) 1073 let alloc_id = alloc_ref.alloc_id; 1074 let bytes = alloc_ref 1075 .alloc 1076 .get_bytes_mut(&alloc_ref.tcx, alloc_ref.range) 1077 .map_err(move |e| e.to_interp_error(alloc_id))?; 1078 // `zip` would stop when the first iterator ends; we want to definitely 1079 // cover all of `bytes`. 1080 for dest in bytes { 1081 *dest = src.next().expect("iterator was shorter than it said it would be"); 1082 } 1083 assert_matches!(src.next(), None, "iterator was longer than it said it would be"); 1084 Ok(()) 1085 } 1086 mem_copy( &mut self, src: Pointer<Option<M::Provenance>>, src_align: Align, dest: Pointer<Option<M::Provenance>>, dest_align: Align, size: Size, nonoverlapping: bool, ) -> InterpResult<'tcx>1087 pub fn mem_copy( 1088 &mut self, 1089 src: Pointer<Option<M::Provenance>>, 1090 src_align: Align, 1091 dest: Pointer<Option<M::Provenance>>, 1092 dest_align: Align, 1093 size: Size, 1094 nonoverlapping: bool, 1095 ) -> InterpResult<'tcx> { 1096 self.mem_copy_repeatedly(src, src_align, dest, dest_align, size, 1, nonoverlapping) 1097 } 1098 mem_copy_repeatedly( &mut self, src: Pointer<Option<M::Provenance>>, src_align: Align, dest: Pointer<Option<M::Provenance>>, dest_align: Align, size: Size, num_copies: u64, nonoverlapping: bool, ) -> InterpResult<'tcx>1099 pub fn mem_copy_repeatedly( 1100 &mut self, 1101 src: Pointer<Option<M::Provenance>>, 1102 src_align: Align, 1103 dest: Pointer<Option<M::Provenance>>, 1104 dest_align: Align, 1105 size: Size, 1106 num_copies: u64, 1107 nonoverlapping: bool, 1108 ) -> InterpResult<'tcx> { 1109 let tcx = self.tcx; 1110 // We need to do our own bounds-checks. 1111 let src_parts = self.get_ptr_access(src, size, src_align)?; 1112 let dest_parts = self.get_ptr_access(dest, size * num_copies, dest_align)?; // `Size` multiplication 1113 1114 // FIXME: we look up both allocations twice here, once before for the `check_ptr_access` 1115 // and once below to get the underlying `&[mut] Allocation`. 1116 1117 // Source alloc preparations and access hooks. 1118 let Some((src_alloc_id, src_offset, src_prov)) = src_parts else { 1119 // Zero-sized *source*, that means dest is also zero-sized and we have nothing to do. 1120 return Ok(()); 1121 }; 1122 let src_alloc = self.get_alloc_raw(src_alloc_id)?; 1123 let src_range = alloc_range(src_offset, size); 1124 M::before_memory_read( 1125 *tcx, 1126 &self.machine, 1127 &src_alloc.extra, 1128 (src_alloc_id, src_prov), 1129 src_range, 1130 )?; 1131 // We need the `dest` ptr for the next operation, so we get it now. 1132 // We already did the source checks and called the hooks so we are good to return early. 1133 let Some((dest_alloc_id, dest_offset, dest_prov)) = dest_parts else { 1134 // Zero-sized *destination*. 1135 return Ok(()); 1136 }; 1137 1138 // Prepare getting source provenance. 1139 let src_bytes = src_alloc.get_bytes_unchecked(src_range).as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation 1140 // first copy the provenance to a temporary buffer, because 1141 // `get_bytes_mut` will clear the provenance, which is correct, 1142 // since we don't want to keep any provenance at the target. 1143 // This will also error if copying partial provenance is not supported. 1144 let provenance = src_alloc 1145 .provenance() 1146 .prepare_copy(src_range, dest_offset, num_copies, self) 1147 .map_err(|e| e.to_interp_error(dest_alloc_id))?; 1148 // Prepare a copy of the initialization mask. 1149 let init = src_alloc.init_mask().prepare_copy(src_range); 1150 1151 // Destination alloc preparations and access hooks. 1152 let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?; 1153 let dest_range = alloc_range(dest_offset, size * num_copies); 1154 M::before_memory_write( 1155 *tcx, 1156 extra, 1157 &mut dest_alloc.extra, 1158 (dest_alloc_id, dest_prov), 1159 dest_range, 1160 )?; 1161 let dest_bytes = dest_alloc 1162 .get_bytes_mut_ptr(&tcx, dest_range) 1163 .map_err(|e| e.to_interp_error(dest_alloc_id))? 1164 .as_mut_ptr(); 1165 1166 if init.no_bytes_init() { 1167 // Fast path: If all bytes are `uninit` then there is nothing to copy. The target range 1168 // is marked as uninitialized but we otherwise omit changing the byte representation which may 1169 // be arbitrary for uninitialized bytes. 1170 // This also avoids writing to the target bytes so that the backing allocation is never 1171 // touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary 1172 // operating system this can avoid physically allocating the page. 1173 dest_alloc 1174 .write_uninit(&tcx, dest_range) 1175 .map_err(|e| e.to_interp_error(dest_alloc_id))?; 1176 // We can forget about the provenance, this is all not initialized anyway. 1177 return Ok(()); 1178 } 1179 1180 // SAFE: The above indexing would have panicked if there weren't at least `size` bytes 1181 // behind `src` and `dest`. Also, we use the overlapping-safe `ptr::copy` if `src` and 1182 // `dest` could possibly overlap. 1183 // The pointers above remain valid even if the `HashMap` table is moved around because they 1184 // point into the `Vec` storing the bytes. 1185 unsafe { 1186 if src_alloc_id == dest_alloc_id { 1187 if nonoverlapping { 1188 // `Size` additions 1189 if (src_offset <= dest_offset && src_offset + size > dest_offset) 1190 || (dest_offset <= src_offset && dest_offset + size > src_offset) 1191 { 1192 throw_ub_custom!(fluent::const_eval_copy_nonoverlapping_overlapping); 1193 } 1194 } 1195 1196 for i in 0..num_copies { 1197 ptr::copy( 1198 src_bytes, 1199 dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication 1200 size.bytes_usize(), 1201 ); 1202 } 1203 } else { 1204 for i in 0..num_copies { 1205 ptr::copy_nonoverlapping( 1206 src_bytes, 1207 dest_bytes.add((size * i).bytes_usize()), // `Size` multiplication 1208 size.bytes_usize(), 1209 ); 1210 } 1211 } 1212 } 1213 1214 // now fill in all the "init" data 1215 dest_alloc.init_mask_apply_copy( 1216 init, 1217 alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`) 1218 num_copies, 1219 ); 1220 // copy the provenance to the destination 1221 dest_alloc.provenance_apply_copy(provenance); 1222 1223 Ok(()) 1224 } 1225 } 1226 1227 /// Machine pointer introspection. 1228 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { 1229 /// Test if this value might be null. 1230 /// If the machine does not support ptr-to-int casts, this is conservative. scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool>1231 pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> { 1232 Ok(match scalar.try_to_int() { 1233 Ok(int) => int.is_null(), 1234 Err(_) => { 1235 // Can only happen during CTFE. 1236 let ptr = scalar.to_pointer(self)?; 1237 match self.ptr_try_get_alloc_id(ptr) { 1238 Ok((alloc_id, offset, _)) => { 1239 let (size, _align, _kind) = self.get_alloc_info(alloc_id); 1240 // If the pointer is out-of-bounds, it may be null. 1241 // Note that one-past-the-end (offset == size) is still inbounds, and never null. 1242 offset > size 1243 } 1244 Err(_offset) => bug!("a non-int scalar is always a pointer"), 1245 } 1246 } 1247 }) 1248 } 1249 1250 /// Turning a "maybe pointer" into a proper pointer (and some information 1251 /// about where it points), or an absolute address. ptr_try_get_alloc_id( &self, ptr: Pointer<Option<M::Provenance>>, ) -> Result<(AllocId, Size, M::ProvenanceExtra), u64>1252 pub fn ptr_try_get_alloc_id( 1253 &self, 1254 ptr: Pointer<Option<M::Provenance>>, 1255 ) -> Result<(AllocId, Size, M::ProvenanceExtra), u64> { 1256 match ptr.into_pointer_or_addr() { 1257 Ok(ptr) => match M::ptr_get_alloc(self, ptr) { 1258 Some((alloc_id, offset, extra)) => Ok((alloc_id, offset, extra)), 1259 None => { 1260 assert!(M::Provenance::OFFSET_IS_ADDR); 1261 let (_, addr) = ptr.into_parts(); 1262 Err(addr.bytes()) 1263 } 1264 }, 1265 Err(addr) => Err(addr.bytes()), 1266 } 1267 } 1268 1269 /// Turning a "maybe pointer" into a proper pointer (and some information about where it points). 1270 #[inline(always)] ptr_get_alloc_id( &self, ptr: Pointer<Option<M::Provenance>>, ) -> InterpResult<'tcx, (AllocId, Size, M::ProvenanceExtra)>1271 pub fn ptr_get_alloc_id( 1272 &self, 1273 ptr: Pointer<Option<M::Provenance>>, 1274 ) -> InterpResult<'tcx, (AllocId, Size, M::ProvenanceExtra)> { 1275 self.ptr_try_get_alloc_id(ptr).map_err(|offset| { 1276 err_ub!(DanglingIntPointer(offset, CheckInAllocMsg::InboundsTest)).into() 1277 }) 1278 } 1279 } 1280