From 8c30fd8d6fca13a2953770adc7b4b3b7f424f2d2 Mon Sep 17 00:00:00 2001 From: rajgandhi1 Date: Wed, 6 May 2026 17:01:30 +0530 Subject: [PATCH 1/3] Guard SizeSkeleton::compute against stack overflow --- compiler/rustc_middle/src/ty/layout.rs | 28 ++++++++++++++++--- tests/ui/transmute/recursive-size-skeleton.rs | 26 +++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 tests/ui/transmute/recursive-size-skeleton.rs diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index 7155df08ec59d..bc70fbb771655 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -332,9 +332,24 @@ impl<'tcx> SizeSkeleton<'tcx> { ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, + ) -> Result, &'tcx LayoutError<'tcx>> { + Self::compute_inner(ty, tcx, typing_env, 0) + } + + fn compute_inner( + ty: Ty<'tcx>, + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + depth: usize, ) -> Result, &'tcx LayoutError<'tcx>> { debug_assert!(!ty.has_non_region_infer()); + /// Bail out if we've recursed too deeply (issue #156137); a cyclic type + /// alias can otherwise blow the stack here. + if !tcx.recursion_limit().value_within_limit(depth) { + return Err(tcx.arena.alloc(LayoutError::Unknown(ty))); + } + // First try computing a static layout. let err = match tcx.layout_of(typing_env.as_query_input(ty)) { Ok(layout) => { @@ -407,7 +422,7 @@ impl<'tcx> SizeSkeleton<'tcx> { return Ok(SizeSkeleton::Known(Size::from_bytes(0), None)); } - match SizeSkeleton::compute(inner, tcx, typing_env)? { + match SizeSkeleton::compute_inner(inner, tcx, typing_env, depth + 1)? { // This may succeed because the multiplication of two types may overflow // but a single size of a nested array will not. SizeSkeleton::Known(s, a) => { @@ -436,7 +451,12 @@ impl<'tcx> SizeSkeleton<'tcx> { let i = VariantIdx::from_usize(i); let fields = def.variant(i).fields.iter().map(|field| { - SizeSkeleton::compute(field.ty(tcx, args), tcx, typing_env) + SizeSkeleton::compute_inner( + field.ty(tcx, args), + tcx, + typing_env, + depth + 1, + ) }); let mut ptr = None; for field in fields { @@ -487,13 +507,13 @@ impl<'tcx> SizeSkeleton<'tcx> { if ty == normalized { Err(err) } else { - SizeSkeleton::compute(normalized, tcx, typing_env) + SizeSkeleton::compute_inner(normalized, tcx, typing_env, depth + 1) } } ty::Pat(base, pat) => { // Pattern types are always the same size as their base. - let base = SizeSkeleton::compute(base, tcx, typing_env); + let base = SizeSkeleton::compute_inner(base, tcx, typing_env, depth + 1); match *pat { ty::PatternKind::Range { .. } | ty::PatternKind::Or(_) => base, // But in the case of `!null` patterns we need to note that in the diff --git a/tests/ui/transmute/recursive-size-skeleton.rs b/tests/ui/transmute/recursive-size-skeleton.rs new file mode 100644 index 0000000000000..6483e0dd53967 --- /dev/null +++ b/tests/ui/transmute/recursive-size-skeleton.rs @@ -0,0 +1,26 @@ +/// Regression test for issue #156137. +/// +/// Computing the `SizeSkeleton` of a type whose layout depends on itself +/// through a normalizing type alias used to recurse without bound and +/// blow the stack. We now bail out via the recursion limit and emit a +/// regular error instead. + +use std::mem::transmute; + +trait Trait { + type Assoc; +} +impl Trait for T { + type Assoc = T; +} +type Alias = ::Assoc; + +pub struct Thing(*const T, Alias>); +/// ERROR recursive type + +pub fn weird(value: Thing) { + let _: i32 = unsafe { transmute(value) }; + /// ERROR cannot transmute between types of different sizes +} + +fn main() {} From ddff79f9be8b9cecc5b1f394213500c8dcac424b Mon Sep 17 00:00:00 2001 From: rajgandhi1 Date: Wed, 6 May 2026 17:40:38 +0530 Subject: [PATCH 2/3] fix formatting. --- compiler/rustc_middle/src/ty/layout.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index bc70fbb771655..2c654355a98e9 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -344,8 +344,8 @@ impl<'tcx> SizeSkeleton<'tcx> { ) -> Result, &'tcx LayoutError<'tcx>> { debug_assert!(!ty.has_non_region_infer()); - /// Bail out if we've recursed too deeply (issue #156137); a cyclic type - /// alias can otherwise blow the stack here. + // Bail out if we've recursed too deeply (issue #156137); a cyclic type + // alias can otherwise blow the stack here. if !tcx.recursion_limit().value_within_limit(depth) { return Err(tcx.arena.alloc(LayoutError::Unknown(ty))); } @@ -449,15 +449,9 @@ impl<'tcx> SizeSkeleton<'tcx> { // Get a zero-sized variant or a pointer newtype. let zero_or_ptr_variant = |i| { let i = VariantIdx::from_usize(i); - let fields = - def.variant(i).fields.iter().map(|field| { - SizeSkeleton::compute_inner( - field.ty(tcx, args), - tcx, - typing_env, - depth + 1, - ) - }); + let fields = def.variant(i).fields.iter().map(|field| { + SizeSkeleton::compute_inner(field.ty(tcx, args), tcx, typing_env, depth + 1) + }); let mut ptr = None; for field in fields { let field = field?; From 866ea899efc809c48abba082ca033bcfa04b40d2 Mon Sep 17 00:00:00 2001 From: rajgandhi1 Date: Wed, 6 May 2026 19:25:10 +0530 Subject: [PATCH 3/3] add compiletest file --- tests/ui/transmute/recursive-size-skeleton.rs | 14 ++++++-------- tests/ui/transmute/recursive-size-skeleton.stderr | 12 ++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 tests/ui/transmute/recursive-size-skeleton.stderr diff --git a/tests/ui/transmute/recursive-size-skeleton.rs b/tests/ui/transmute/recursive-size-skeleton.rs index 6483e0dd53967..4392fa0d65077 100644 --- a/tests/ui/transmute/recursive-size-skeleton.rs +++ b/tests/ui/transmute/recursive-size-skeleton.rs @@ -1,9 +1,8 @@ -/// Regression test for issue #156137. -/// -/// Computing the `SizeSkeleton` of a type whose layout depends on itself -/// through a normalizing type alias used to recurse without bound and -/// blow the stack. We now bail out via the recursion limit and emit a -/// regular error instead. +// Regression test for issue #156137. +// Computing the `SizeSkeleton` of a type whose layout depends on itself +// through a normalizing type alias used to recurse without bound and +// blow the stack. We now bail out via the recursion limit and emit a +// regular error instead of ICE-ing. use std::mem::transmute; @@ -16,11 +15,10 @@ impl Trait for T { type Alias = ::Assoc; pub struct Thing(*const T, Alias>); -/// ERROR recursive type pub fn weird(value: Thing) { let _: i32 = unsafe { transmute(value) }; - /// ERROR cannot transmute between types of different sizes + //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types } fn main() {} diff --git a/tests/ui/transmute/recursive-size-skeleton.stderr b/tests/ui/transmute/recursive-size-skeleton.stderr new file mode 100644 index 0000000000000..f8903b58d36ca --- /dev/null +++ b/tests/ui/transmute/recursive-size-skeleton.stderr @@ -0,0 +1,12 @@ +error[E0512]: cannot transmute between types of different sizes, or dependently-sized types + --> $DIR/recursive-size-skeleton.rs:20:27 + | +LL | let _: i32 = unsafe { transmute(value) }; + | ^^^^^^^^^ + | + = note: source type: `Thing` (the type `*const T` has an unknown layout) + = note: target type: `i32` (32 bits) + +error: aborting due to 1 previous error + +For more information about this error, try `rustc --explain E0512`.