1 use super::eval_queries::{mk_eval_cx, op_to_const};
2 use super::machine::CompileTimeEvalContext;
3 use super::{ValTreeCreationError, ValTreeCreationResult, VALTREE_MAX_NODES};
4 use crate::const_eval::CanAccessStatics;
5 use crate::interpret::{
6 intern_const_alloc_recursive, ConstValue, ImmTy, Immediate, InternKind, MemPlaceMeta,
7 MemoryKind, PlaceTy, Scalar,
8 };
9 use crate::interpret::{MPlaceTy, Value};
10 use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
11 use rustc_span::source_map::DUMMY_SP;
12 use rustc_target::abi::{Align, FieldIdx, VariantIdx, FIRST_VARIANT};
13
14 #[instrument(skip(ecx), level = "debug")]
branches<'tcx>( ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: &MPlaceTy<'tcx>, n: usize, variant: Option<VariantIdx>, num_nodes: &mut usize, ) -> ValTreeCreationResult<'tcx>15 fn branches<'tcx>(
16 ecx: &CompileTimeEvalContext<'tcx, 'tcx>,
17 place: &MPlaceTy<'tcx>,
18 n: usize,
19 variant: Option<VariantIdx>,
20 num_nodes: &mut usize,
21 ) -> ValTreeCreationResult<'tcx> {
22 let place = match variant {
23 Some(variant) => ecx.mplace_downcast(&place, variant).unwrap(),
24 None => *place,
25 };
26 let variant = variant.map(|variant| Some(ty::ValTree::Leaf(ScalarInt::from(variant.as_u32()))));
27 debug!(?place, ?variant);
28
29 let mut fields = Vec::with_capacity(n);
30 for i in 0..n {
31 let field = ecx.mplace_field(&place, i).unwrap();
32 let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?;
33 fields.push(Some(valtree));
34 }
35
36 // For enums, we prepend their variant index before the variant's fields so we can figure out
37 // the variant again when just seeing a valtree.
38 let branches = variant
39 .into_iter()
40 .chain(fields.into_iter())
41 .collect::<Option<Vec<_>>>()
42 .expect("should have already checked for errors in ValTree creation");
43
44 // Have to account for ZSTs here
45 if branches.len() == 0 {
46 *num_nodes += 1;
47 }
48
49 Ok(ty::ValTree::Branch(ecx.tcx.arena.alloc_from_iter(branches)))
50 }
51
52 #[instrument(skip(ecx), level = "debug")]
slice_branches<'tcx>( ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: &MPlaceTy<'tcx>, num_nodes: &mut usize, ) -> ValTreeCreationResult<'tcx>53 fn slice_branches<'tcx>(
54 ecx: &CompileTimeEvalContext<'tcx, 'tcx>,
55 place: &MPlaceTy<'tcx>,
56 num_nodes: &mut usize,
57 ) -> ValTreeCreationResult<'tcx> {
58 let n = place
59 .len(&ecx.tcx.tcx)
60 .unwrap_or_else(|_| panic!("expected to use len of place {:?}", place));
61
62 let mut elems = Vec::with_capacity(n as usize);
63 for i in 0..n {
64 let place_elem = ecx.mplace_index(place, i).unwrap();
65 let valtree = const_to_valtree_inner(ecx, &place_elem, num_nodes)?;
66 elems.push(valtree);
67 }
68
69 Ok(ty::ValTree::Branch(ecx.tcx.arena.alloc_from_iter(elems)))
70 }
71
72 #[instrument(skip(ecx), level = "debug")]
const_to_valtree_inner<'tcx>( ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: &MPlaceTy<'tcx>, num_nodes: &mut usize, ) -> ValTreeCreationResult<'tcx>73 pub(crate) fn const_to_valtree_inner<'tcx>(
74 ecx: &CompileTimeEvalContext<'tcx, 'tcx>,
75 place: &MPlaceTy<'tcx>,
76 num_nodes: &mut usize,
77 ) -> ValTreeCreationResult<'tcx> {
78 let ty = place.layout.ty;
79 debug!("ty kind: {:?}", ty.kind());
80
81 if *num_nodes >= VALTREE_MAX_NODES {
82 return Err(ValTreeCreationError::NodesOverflow);
83 }
84
85 match ty.kind() {
86 ty::FnDef(..) => {
87 *num_nodes += 1;
88 Ok(ty::ValTree::zst())
89 }
90 ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
91 let Ok(val) = ecx.read_immediate(&place.into()) else {
92 return Err(ValTreeCreationError::Other);
93 };
94 let val = val.to_scalar();
95 *num_nodes += 1;
96
97 Ok(ty::ValTree::Leaf(val.assert_int()))
98 }
99
100 // Raw pointers are not allowed in type level constants, as we cannot properly test them for
101 // equality at compile-time (see `ptr_guaranteed_cmp`).
102 // Technically we could allow function pointers (represented as `ty::Instance`), but this is not guaranteed to
103 // agree with runtime equality tests.
104 ty::FnPtr(_) | ty::RawPtr(_) => Err(ValTreeCreationError::NonSupportedType),
105
106 ty::Ref(_, _, _) => {
107 let Ok(derefd_place)= ecx.deref_operand(&place.into()) else {
108 return Err(ValTreeCreationError::Other);
109 };
110 debug!(?derefd_place);
111
112 const_to_valtree_inner(ecx, &derefd_place, num_nodes)
113 }
114
115 ty::Str | ty::Slice(_) | ty::Array(_, _) => {
116 slice_branches(ecx, place, num_nodes)
117 }
118 // Trait objects are not allowed in type level constants, as we have no concept for
119 // resolving their backing type, even if we can do that at const eval time. We may
120 // hypothetically be able to allow `dyn StructuralEq` trait objects in the future,
121 // but it is unclear if this is useful.
122 ty::Dynamic(..) => Err(ValTreeCreationError::NonSupportedType),
123
124 ty::Tuple(elem_tys) => {
125 branches(ecx, place, elem_tys.len(), None, num_nodes)
126 }
127
128 ty::Adt(def, _) => {
129 if def.is_union() {
130 return Err(ValTreeCreationError::NonSupportedType);
131 } else if def.variants().is_empty() {
132 bug!("uninhabited types should have errored and never gotten converted to valtree")
133 }
134
135 let Ok((_, variant)) = ecx.read_discriminant(&place.into()) else {
136 return Err(ValTreeCreationError::Other);
137 };
138 branches(ecx, place, def.variant(variant).fields.len(), def.is_enum().then_some(variant), num_nodes)
139 }
140
141 ty::Never
142 | ty::Error(_)
143 | ty::Foreign(..)
144 | ty::Infer(ty::FreshIntTy(_))
145 | ty::Infer(ty::FreshFloatTy(_))
146 // FIXME(oli-obk): we could look behind opaque types
147 | ty::Alias(..)
148 | ty::Param(_)
149 | ty::Bound(..)
150 | ty::Placeholder(..)
151 | ty::Infer(_)
152 // FIXME(oli-obk): we can probably encode closures just like structs
153 | ty::Closure(..)
154 | ty::Generator(..)
155 | ty::GeneratorWitness(..) |ty::GeneratorWitnessMIR(..)=> Err(ValTreeCreationError::NonSupportedType),
156 }
157 }
158
159 #[instrument(skip(ecx), level = "debug")]
create_mplace_from_layout<'tcx>( ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, ty: Ty<'tcx>, ) -> MPlaceTy<'tcx>160 fn create_mplace_from_layout<'tcx>(
161 ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>,
162 ty: Ty<'tcx>,
163 ) -> MPlaceTy<'tcx> {
164 let tcx = ecx.tcx;
165 let param_env = ecx.param_env;
166 let layout = tcx.layout_of(param_env.and(ty)).unwrap();
167 debug!(?layout);
168
169 ecx.allocate(layout, MemoryKind::Stack).unwrap()
170 }
171
172 // Walks custom DSTs and gets the type of the unsized field and the number of elements
173 // in the unsized field.
get_info_on_unsized_field<'tcx>( ty: Ty<'tcx>, valtree: ty::ValTree<'tcx>, tcx: TyCtxt<'tcx>, ) -> (Ty<'tcx>, usize)174 fn get_info_on_unsized_field<'tcx>(
175 ty: Ty<'tcx>,
176 valtree: ty::ValTree<'tcx>,
177 tcx: TyCtxt<'tcx>,
178 ) -> (Ty<'tcx>, usize) {
179 let mut last_valtree = valtree;
180 let tail = tcx.struct_tail_with_normalize(
181 ty,
182 |ty| ty,
183 || {
184 let branches = last_valtree.unwrap_branch();
185 last_valtree = branches[branches.len() - 1];
186 debug!(?branches, ?last_valtree);
187 },
188 );
189 let unsized_inner_ty = match tail.kind() {
190 ty::Slice(t) => *t,
191 ty::Str => tail,
192 _ => bug!("expected Slice or Str"),
193 };
194
195 // Have to adjust type for ty::Str
196 let unsized_inner_ty = match unsized_inner_ty.kind() {
197 ty::Str => tcx.types.u8,
198 _ => unsized_inner_ty,
199 };
200
201 // Get the number of elements in the unsized field
202 let num_elems = last_valtree.unwrap_branch().len();
203
204 (unsized_inner_ty, num_elems)
205 }
206
207 #[instrument(skip(ecx), level = "debug", ret)]
create_pointee_place<'tcx>( ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, ty: Ty<'tcx>, valtree: ty::ValTree<'tcx>, ) -> MPlaceTy<'tcx>208 fn create_pointee_place<'tcx>(
209 ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>,
210 ty: Ty<'tcx>,
211 valtree: ty::ValTree<'tcx>,
212 ) -> MPlaceTy<'tcx> {
213 let tcx = ecx.tcx.tcx;
214
215 if !ty.is_sized(*ecx.tcx, ty::ParamEnv::empty()) {
216 // We need to create `Allocation`s for custom DSTs
217
218 let (unsized_inner_ty, num_elems) = get_info_on_unsized_field(ty, valtree, tcx);
219 let unsized_inner_ty = match unsized_inner_ty.kind() {
220 ty::Str => tcx.types.u8,
221 _ => unsized_inner_ty,
222 };
223 let unsized_inner_ty_size =
224 tcx.layout_of(ty::ParamEnv::empty().and(unsized_inner_ty)).unwrap().layout.size();
225 debug!(?unsized_inner_ty, ?unsized_inner_ty_size, ?num_elems);
226
227 // for custom DSTs only the last field/element is unsized, but we need to also allocate
228 // space for the other fields/elements
229 let layout = tcx.layout_of(ty::ParamEnv::empty().and(ty)).unwrap();
230 let size_of_sized_part = layout.layout.size();
231
232 // Get the size of the memory behind the DST
233 let dst_size = unsized_inner_ty_size.checked_mul(num_elems as u64, &tcx).unwrap();
234
235 let size = size_of_sized_part.checked_add(dst_size, &tcx).unwrap();
236 let align = Align::from_bytes(size.bytes().next_power_of_two()).unwrap();
237 let ptr = ecx.allocate_ptr(size, align, MemoryKind::Stack).unwrap();
238 debug!(?ptr);
239
240 MPlaceTy::from_aligned_ptr_with_meta(
241 ptr.into(),
242 layout,
243 MemPlaceMeta::Meta(Scalar::from_target_usize(num_elems as u64, &tcx)),
244 )
245 } else {
246 create_mplace_from_layout(ecx, ty)
247 }
248 }
249
250 /// Converts a `ValTree` to a `ConstValue`, which is needed after mir
251 /// construction has finished.
252 // FIXME Merge `valtree_to_const_value` and `valtree_into_mplace` into one function
253 #[instrument(skip(tcx), level = "debug", ret)]
valtree_to_const_value<'tcx>( tcx: TyCtxt<'tcx>, param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, valtree: ty::ValTree<'tcx>, ) -> ConstValue<'tcx>254 pub fn valtree_to_const_value<'tcx>(
255 tcx: TyCtxt<'tcx>,
256 param_env_ty: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
257 valtree: ty::ValTree<'tcx>,
258 ) -> ConstValue<'tcx> {
259 // Basic idea: We directly construct `Scalar` values from trivial `ValTree`s
260 // (those for constants with type bool, int, uint, float or char).
261 // For all other types we create an `MPlace` and fill that by walking
262 // the `ValTree` and using `place_projection` and `place_field` to
263 // create inner `MPlace`s which are filled recursively.
264 // FIXME Does this need an example?
265
266 let (param_env, ty) = param_env_ty.into_parts();
267 let mut ecx: crate::interpret::InterpCx<
268 '_,
269 '_,
270 crate::const_eval::CompileTimeInterpreter<'_, '_>,
271 > = mk_eval_cx(tcx, DUMMY_SP, param_env, CanAccessStatics::No);
272
273 match ty.kind() {
274 ty::FnDef(..) => {
275 assert!(valtree.unwrap_branch().is_empty());
276 ConstValue::ZeroSized
277 }
278 ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => match valtree {
279 ty::ValTree::Leaf(scalar_int) => ConstValue::Scalar(Scalar::Int(scalar_int)),
280 ty::ValTree::Branch(_) => bug!(
281 "ValTrees for Bool, Int, Uint, Float or Char should have the form ValTree::Leaf"
282 ),
283 },
284 ty::Ref(_, _, _) | ty::Tuple(_) | ty::Array(_, _) | ty::Adt(..) => {
285 let mut place = match ty.kind() {
286 ty::Ref(_, inner_ty, _) => {
287 // Need to create a place for the pointee to fill for Refs
288 create_pointee_place(&mut ecx, *inner_ty, valtree)
289 }
290 _ => create_mplace_from_layout(&mut ecx, ty),
291 };
292 debug!(?place);
293
294 valtree_into_mplace(&mut ecx, &mut place, valtree);
295 dump_place(&ecx, place.into());
296 intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &place).unwrap();
297
298 match ty.kind() {
299 ty::Ref(_, _, _) => {
300 let ref_place = place.to_ref(&tcx);
301 let imm =
302 ImmTy::from_immediate(ref_place, tcx.layout_of(param_env_ty).unwrap());
303
304 op_to_const(&ecx, &imm.into())
305 }
306 _ => op_to_const(&ecx, &place.into()),
307 }
308 }
309 ty::Never
310 | ty::Error(_)
311 | ty::Foreign(..)
312 | ty::Infer(ty::FreshIntTy(_))
313 | ty::Infer(ty::FreshFloatTy(_))
314 | ty::Alias(..)
315 | ty::Param(_)
316 | ty::Bound(..)
317 | ty::Placeholder(..)
318 | ty::Infer(_)
319 | ty::Closure(..)
320 | ty::Generator(..)
321 | ty::GeneratorWitness(..)
322 | ty::GeneratorWitnessMIR(..)
323 | ty::FnPtr(_)
324 | ty::RawPtr(_)
325 | ty::Str
326 | ty::Slice(_)
327 | ty::Dynamic(..) => bug!("no ValTree should have been created for type {:?}", ty.kind()),
328 }
329 }
330
331 #[instrument(skip(ecx), level = "debug")]
valtree_into_mplace<'tcx>( ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>, place: &mut MPlaceTy<'tcx>, valtree: ty::ValTree<'tcx>, )332 fn valtree_into_mplace<'tcx>(
333 ecx: &mut CompileTimeEvalContext<'tcx, 'tcx>,
334 place: &mut MPlaceTy<'tcx>,
335 valtree: ty::ValTree<'tcx>,
336 ) {
337 // This will match on valtree and write the value(s) corresponding to the ValTree
338 // inside the place recursively.
339
340 let tcx = ecx.tcx.tcx;
341 let ty = place.layout.ty;
342
343 match ty.kind() {
344 ty::FnDef(_, _) => {
345 // Zero-sized type, nothing to do.
346 }
347 ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
348 let scalar_int = valtree.unwrap_leaf();
349 debug!("writing trivial valtree {:?} to place {:?}", scalar_int, place);
350 ecx.write_immediate(Immediate::Scalar(scalar_int.into()), &place.into()).unwrap();
351 }
352 ty::Ref(_, inner_ty, _) => {
353 let mut pointee_place = create_pointee_place(ecx, *inner_ty, valtree);
354 debug!(?pointee_place);
355
356 valtree_into_mplace(ecx, &mut pointee_place, valtree);
357 dump_place(ecx, pointee_place.into());
358 intern_const_alloc_recursive(ecx, InternKind::Constant, &pointee_place).unwrap();
359
360 let imm = match inner_ty.kind() {
361 ty::Slice(_) | ty::Str => {
362 let len = valtree.unwrap_branch().len();
363 let len_scalar = Scalar::from_target_usize(len as u64, &tcx);
364
365 Immediate::ScalarPair(
366 Scalar::from_maybe_pointer((*pointee_place).ptr, &tcx),
367 len_scalar,
368 )
369 }
370 _ => pointee_place.to_ref(&tcx),
371 };
372 debug!(?imm);
373
374 ecx.write_immediate(imm, &place.into()).unwrap();
375 }
376 ty::Adt(_, _) | ty::Tuple(_) | ty::Array(_, _) | ty::Str | ty::Slice(_) => {
377 let branches = valtree.unwrap_branch();
378
379 // Need to downcast place for enums
380 let (place_adjusted, branches, variant_idx) = match ty.kind() {
381 ty::Adt(def, _) if def.is_enum() => {
382 // First element of valtree corresponds to variant
383 let scalar_int = branches[0].unwrap_leaf();
384 let variant_idx = VariantIdx::from_u32(scalar_int.try_to_u32().unwrap());
385 let variant = def.variant(variant_idx);
386 debug!(?variant);
387
388 (
389 place.project_downcast(ecx, variant_idx).unwrap(),
390 &branches[1..],
391 Some(variant_idx),
392 )
393 }
394 _ => (*place, branches, None),
395 };
396 debug!(?place_adjusted, ?branches);
397
398 // Create the places (by indexing into `place`) for the fields and fill
399 // them recursively
400 for (i, inner_valtree) in branches.iter().enumerate() {
401 debug!(?i, ?inner_valtree);
402
403 let mut place_inner = match ty.kind() {
404 ty::Str | ty::Slice(_) => ecx.mplace_index(&place, i as u64).unwrap(),
405 _ if !ty.is_sized(*ecx.tcx, ty::ParamEnv::empty())
406 && i == branches.len() - 1 =>
407 {
408 // Note: For custom DSTs we need to manually process the last unsized field.
409 // We created a `Pointer` for the `Allocation` of the complete sized version of
410 // the Adt in `create_pointee_place` and now we fill that `Allocation` with the
411 // values in the ValTree. For the unsized field we have to additionally add the meta
412 // data.
413
414 let (unsized_inner_ty, num_elems) =
415 get_info_on_unsized_field(ty, valtree, tcx);
416 debug!(?unsized_inner_ty);
417
418 let inner_ty = match ty.kind() {
419 ty::Adt(def, substs) => {
420 let i = FieldIdx::from_usize(i);
421 def.variant(FIRST_VARIANT).fields[i].ty(tcx, substs)
422 }
423 ty::Tuple(inner_tys) => inner_tys[i],
424 _ => bug!("unexpected unsized type {:?}", ty),
425 };
426
427 let inner_layout =
428 tcx.layout_of(ty::ParamEnv::empty().and(inner_ty)).unwrap();
429 debug!(?inner_layout);
430
431 let offset = place_adjusted.layout.fields.offset(i);
432 place
433 .offset_with_meta(
434 offset,
435 MemPlaceMeta::Meta(Scalar::from_target_usize(
436 num_elems as u64,
437 &tcx,
438 )),
439 inner_layout,
440 &tcx,
441 )
442 .unwrap()
443 }
444 _ => ecx.mplace_field(&place_adjusted, i).unwrap(),
445 };
446
447 debug!(?place_inner);
448 valtree_into_mplace(ecx, &mut place_inner, *inner_valtree);
449 dump_place(&ecx, place_inner.into());
450 }
451
452 debug!("dump of place_adjusted:");
453 dump_place(ecx, place_adjusted.into());
454
455 if let Some(variant_idx) = variant_idx {
456 // don't forget filling the place with the discriminant of the enum
457 ecx.write_discriminant(variant_idx, &place.into()).unwrap();
458 }
459
460 debug!("dump of place after writing discriminant:");
461 dump_place(ecx, place.into());
462 }
463 _ => bug!("shouldn't have created a ValTree for {:?}", ty),
464 }
465 }
466
dump_place<'tcx>(ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: PlaceTy<'tcx>)467 fn dump_place<'tcx>(ecx: &CompileTimeEvalContext<'tcx, 'tcx>, place: PlaceTy<'tcx>) {
468 trace!("{:?}", ecx.dump_place(*place));
469 }
470