From fdd8554059ec6dbba1d2c7aafb4d6c730d52cb90 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Sun, 10 May 2026 04:21:39 +0000 Subject: [PATCH 1/5] TEST: Slim try-build for PGO/BOLT-free measurement --- src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh index 46d34cd001a95..c26ce70a46f10 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh @@ -2,15 +2,12 @@ set -eux -python3 ../x.py build --set rust.debug=true opt-dist - -./build/$HOSTS/stage1-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ +# TESTING-ONLY CHANGES. +touch github-summary.md +python3 ../x.py dist \ --host $HOSTS --target $HOSTS \ - --include-default-paths \ - build-manifest \ - bootstrap \ - enzyme \ - rustc_codegen_gcc + --set "rust.codegen-backends=['llvm']" \ + rustc rust-std cargo rust-src # Use GCC for building GCC components, as it seems to behave badly when built with Clang # Only build GCC on full builds, not try builds From 9da3a8ee37e8296a1691d58ce1df500633d16bb3 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Fri, 8 May 2026 19:39:50 +0000 Subject: [PATCH 2/5] Add `inline` to trivial cross-crate accessors Small accessor functions aren't inlined across crate boundaries in all build configurations. Even with thin LTO, PGO, etc., the backend may be limited in what it's able and willing to inline without these annotations. In detailed profiling, the `inline` attributes being added in this PR seemed to make a difference. Let's add them. Notably, some of these fall within the expansion of the `newtype_index!` macro and will apply to the items it defines. Being in the macro might explain (if later profiling confirms this win) why these weren't noticed earlier. This commit covers the cases where the auto-inline heuristic from Rust PR 116505 is unlikely to work. --- compiler/rustc_index_macros/src/newtype.rs | 7 +++++++ compiler/rustc_middle/src/ty/adt.rs | 2 ++ compiler/rustc_middle/src/ty/consts.rs | 1 + compiler/rustc_middle/src/ty/context.rs | 1 + compiler/rustc_middle/src/ty/predicate.rs | 10 ++++++++++ compiler/rustc_middle/src/ty/region.rs | 1 + compiler/rustc_middle/src/ty/structural_impls.rs | 3 +++ compiler/rustc_middle/src/ty/sty.rs | 2 ++ compiler/rustc_type_ir/src/binder.rs | 8 ++++++++ compiler/rustc_type_ir/src/predicate.rs | 6 ++++++ 10 files changed, 41 insertions(+) diff --git a/compiler/rustc_index_macros/src/newtype.rs b/compiler/rustc_index_macros/src/newtype.rs index 4c56c4fdcd08f..6713761e57ad5 100644 --- a/compiler/rustc_index_macros/src/newtype.rs +++ b/compiler/rustc_index_macros/src/newtype.rs @@ -101,12 +101,14 @@ impl Parse for Newtype { quote! { #gate_rustc_only impl ::rustc_serialize::Decodable for #name { + #[inline] fn decode(d: &mut D) -> Self { Self::from_u32(d.read_u32()) } } #gate_rustc_only impl ::rustc_serialize::Encodable for #name { + #[inline] fn encode(&self, e: &mut E) { e.emit_u32(self.as_u32()); } @@ -138,11 +140,13 @@ impl Parse for Newtype { } } impl ::std::cmp::Ord for #name { + #[inline] fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.as_u32().cmp(&other.as_u32()) } } impl ::std::cmp::PartialOrd for #name { + #[inline] fn partial_cmp(&self, other: &Self) -> Option { self.as_u32().partial_cmp(&other.as_u32()) } @@ -156,6 +160,7 @@ impl Parse for Newtype { quote! { #gate_rustc_only impl ::rustc_data_structures::stable_hasher::StableHash for #name { + #[inline] fn stable_hash< __Hcx: ::rustc_data_structures::stable_hasher::StableHashCtxt >( @@ -340,6 +345,7 @@ impl Parse for Newtype { impl ::std::cmp::Eq for #name {} impl ::std::cmp::PartialEq for #name { + #[inline] fn eq(&self, other: &Self) -> bool { self.as_u32().eq(&other.as_u32()) } @@ -349,6 +355,7 @@ impl Parse for Newtype { impl ::std::marker::StructuralPartialEq for #name {} impl ::std::hash::Hash for #name { + #[inline] fn hash(&self, state: &mut H) { self.as_u32().hash(state) } diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 6381a5c49835d..1eb099acce70a 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -263,10 +263,12 @@ impl<'tcx> rustc_type_ir::inherent::AdtDef> for AdtDef<'tcx> { self.is_struct() } + #[inline] fn is_packed(self) -> bool { self.repr().packed() } + #[inline] fn struct_tail_ty(self, interner: TyCtxt<'tcx>) -> Option>> { Some(interner.type_of(self.non_enum_variant().tail_opt()?.did)) } diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 7421024bba0cc..0a7a20496b262 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -178,6 +178,7 @@ impl<'tcx> rustc_type_ir::inherent::Const> for Const<'tcx> { Const::new_bound(interner, debruijn, bound_const) } + #[inline] fn new_anon_bound(tcx: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self { Const::new_bound(tcx, debruijn, ty::BoundConst::new(var)) } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 33f676e1be64b..bb8a342ba950b 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -114,6 +114,7 @@ impl<'tcx> rustc_type_ir::inherent::Features> for &'tcx rustc_featu self.coroutine_clone() } + #[inline] fn feature_bound_holds_in_crate(self, symbol: Symbol) -> bool { // We don't consider feature bounds to hold in the crate when `staged_api` feature is // enabled, even if it is enabled through `#[feature]`. diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs index 26108f7be6964..d776bd4185a3d 100644 --- a/compiler/rustc_middle/src/ty/predicate.rs +++ b/compiler/rustc_middle/src/ty/predicate.rs @@ -156,10 +156,12 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Clause<'tcx> { } impl<'tcx> Clause<'tcx> { + #[inline] pub fn as_predicate(self) -> Predicate<'tcx> { Predicate(self.0) } + #[inline] pub fn kind(self) -> ty::Binder<'tcx, ClauseKind<'tcx>> { self.0.internee.map_bound(|kind| match kind { PredicateKind::Clause(clause) => clause, @@ -167,6 +169,7 @@ impl<'tcx> Clause<'tcx> { }) } + #[inline] pub fn as_trait_clause(self) -> Option>> { let clause = self.kind(); if let ty::ClauseKind::Trait(trait_clause) = clause.skip_binder() { @@ -176,6 +179,7 @@ impl<'tcx> Clause<'tcx> { } } + #[inline] pub fn as_projection_clause(self) -> Option>> { let clause = self.kind(); if let ty::ClauseKind::Projection(projection_clause) = clause.skip_binder() { @@ -185,6 +189,7 @@ impl<'tcx> Clause<'tcx> { } } + #[inline] pub fn as_type_outlives_clause(self) -> Option>> { let clause = self.kind(); if let ty::ClauseKind::TypeOutlives(o) = clause.skip_binder() { @@ -194,6 +199,7 @@ impl<'tcx> Clause<'tcx> { } } + #[inline] pub fn as_region_outlives_clause( self, ) -> Option>> { @@ -603,6 +609,7 @@ impl<'tcx> UpcastFrom, NormalizesTo<'tcx>> for Predicate<'tcx> { } impl<'tcx> Predicate<'tcx> { + #[inline] pub fn as_trait_clause(self) -> Option> { let predicate = self.kind(); match predicate.skip_binder() { @@ -611,6 +618,7 @@ impl<'tcx> Predicate<'tcx> { } } + #[inline] pub fn as_projection_clause(self) -> Option> { let predicate = self.kind(); match predicate.skip_binder() { @@ -620,6 +628,7 @@ impl<'tcx> Predicate<'tcx> { } /// Matches a `PredicateKind::Clause` and turns it into a `Clause`, otherwise returns `None`. + #[inline] pub fn as_clause(self) -> Option> { match self.kind().skip_binder() { PredicateKind::Clause(..) => Some(self.expect_clause()), @@ -628,6 +637,7 @@ impl<'tcx> Predicate<'tcx> { } /// Assert that the predicate is a clause. + #[inline] pub fn expect_clause(self) -> Clause<'tcx> { match self.kind().skip_binder() { PredicateKind::Clause(..) => Clause(self.0), diff --git a/compiler/rustc_middle/src/ty/region.rs b/compiler/rustc_middle/src/ty/region.rs index 041c6201c71bf..bd7964a9fafcd 100644 --- a/compiler/rustc_middle/src/ty/region.rs +++ b/compiler/rustc_middle/src/ty/region.rs @@ -29,6 +29,7 @@ impl<'tcx> rustc_type_ir::Flags for Region<'tcx> { self.type_flags() } + #[inline] fn outer_exclusive_binder(&self) -> ty::DebruijnIndex { match self.kind() { ty::ReBound(ty::BoundVarIndexKind::Bound(debruijn), _) => debruijn.shifted_in(1), diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 08a8491bab6c5..20094f7a8677d 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -509,6 +509,7 @@ impl<'tcx> TypeFoldable> for ty::Predicate<'tcx> { // FIXME(clause): This is wonky impl<'tcx> TypeFoldable> for ty::Clause<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -516,6 +517,7 @@ impl<'tcx> TypeFoldable> for ty::Clause<'tcx> { Ok(folder.try_fold_predicate(self.as_predicate())?.expect_clause()) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_predicate(self.as_predicate()).expect_clause() } @@ -541,6 +543,7 @@ impl<'tcx> TypeVisitable> for ty::Predicate<'tcx> { } impl<'tcx> TypeVisitable> for ty::Clause<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_predicate(self.as_predicate()) } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 112ad65e19bff..fd95d58e9f3ad 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2146,10 +2146,12 @@ impl<'tcx> Ty<'tcx> { } impl<'tcx> rustc_type_ir::inherent::Tys> for &'tcx ty::List> { + #[inline] fn inputs(self) -> &'tcx [Ty<'tcx>] { self.split_last().unwrap().1 } + #[inline] fn output(self) -> Ty<'tcx> { *self.split_last().unwrap().0 } diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs index de3e04626f823..ab9428571e964 100644 --- a/compiler/rustc_type_ir/src/binder.rs +++ b/compiler/rustc_type_ir/src/binder.rs @@ -109,6 +109,7 @@ where Binder { value, bound_vars: Default::default() } } + #[inline] pub fn bind_with_vars(value: T, bound_vars: I::BoundVarKinds) -> Binder { if cfg!(debug_assertions) { let mut validator = ValidateBoundVars::new(bound_vars); @@ -186,6 +187,7 @@ impl Binder { Binder { value: &self.value, bound_vars: self.bound_vars } } + #[inline] pub fn map_bound_ref>(&self, f: F) -> Binder where F: FnOnce(&T) -> U, @@ -193,6 +195,7 @@ impl Binder { self.as_ref().map_bound(f) } + #[inline] pub fn map_bound>(self, f: F) -> Binder where F: FnOnce(T) -> U, @@ -206,6 +209,7 @@ impl Binder { Binder { value, bound_vars } } + #[inline] pub fn try_map_bound, E>(self, f: F) -> Result, E> where F: FnOnce(T) -> Result, @@ -245,6 +249,7 @@ impl Binder { /// binders, but that would require adjusting the debruijn /// indices, and given the shallow binding structure we often use, /// would not be that useful.) + #[inline] pub fn no_bound_vars(self) -> Option where T: TypeVisitable, @@ -396,6 +401,7 @@ impl EarlyBinder { EarlyBinder { value: &self.value, _tcx: PhantomData } } + #[inline] pub fn map_bound_ref(&self, f: F) -> EarlyBinder where F: FnOnce(&T) -> U, @@ -403,6 +409,7 @@ impl EarlyBinder { self.as_ref().map_bound(f) } + #[inline] pub fn map_bound(self, f: F) -> EarlyBinder where F: FnOnce(T) -> U, @@ -411,6 +418,7 @@ impl EarlyBinder { EarlyBinder { value, _tcx: PhantomData } } + #[inline] pub fn try_map_bound(self, f: F) -> Result, E> where F: FnOnce(T) -> Result, diff --git a/compiler/rustc_type_ir/src/predicate.rs b/compiler/rustc_type_ir/src/predicate.rs index 301cf7dbf1087..fc62348a35d1b 100644 --- a/compiler/rustc_type_ir/src/predicate.rs +++ b/compiler/rustc_type_ir/src/predicate.rs @@ -180,6 +180,7 @@ impl TraitRef { } impl ty::Binder> { + #[inline] pub fn self_ty(&self) -> ty::Binder { self.map_bound_ref(|tr| tr.self_ty()) } @@ -233,11 +234,13 @@ impl TraitPredicate { } impl ty::Binder> { + #[inline] pub fn def_id(self) -> I::TraitId { // Ok to skip binder since trait `DefId` does not care about regions. self.skip_binder().def_id() } + #[inline] pub fn self_ty(self) -> ty::Binder { self.map_bound(|trait_ref| trait_ref.self_ty()) } @@ -946,6 +949,7 @@ impl ty::Binder> { /// /// Note that this is not the `DefId` of the `TraitRef` containing this /// associated type, which is in `tcx.associated_item(projection_def_id()).container`. + #[inline] pub fn item_def_id(&self) -> I::DefId { // Ok to skip binder since trait `DefId` does not care about regions. self.skip_binder().projection_term.def_id() @@ -1025,11 +1029,13 @@ impl HostEffectPredicate { } impl ty::Binder> { + #[inline] pub fn def_id(self) -> I::TraitId { // Ok to skip binder since trait `DefId` does not care about regions. self.skip_binder().def_id() } + #[inline] pub fn self_ty(self) -> ty::Binder { self.map_bound(|trait_ref| trait_ref.self_ty()) } From bb4e9844b9a53e28b19ffde7ac770ca7e5bc0dbd Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Fri, 8 May 2026 19:39:58 +0000 Subject: [PATCH 3/5] Add `inline` to single-call accessor trampolines Some trivial accessors have a single call, like `fn def_id(self) -> DefId { self.did() }`. After MIR inlining the inner call, the auto-inline heuristic from Rust PR 116505 might already cover these, but the heuristic is conservative. Let's annotate explicitly and profile. --- compiler/rustc_middle/src/ty/adt.rs | 7 ++++ compiler/rustc_middle/src/ty/consts.rs | 9 +++++ compiler/rustc_middle/src/ty/context.rs | 7 ++++ compiler/rustc_middle/src/ty/generic_args.rs | 7 ++++ compiler/rustc_middle/src/ty/generics.rs | 1 + compiler/rustc_middle/src/ty/list.rs | 2 ++ compiler/rustc_middle/src/ty/mod.rs | 5 +++ compiler/rustc_middle/src/ty/predicate.rs | 5 +++ compiler/rustc_middle/src/ty/region.rs | 5 +++ .../rustc_middle/src/ty/structural_impls.rs | 15 +++++++++ compiler/rustc_middle/src/ty/sty.rs | 33 +++++++++++++++++++ compiler/rustc_type_ir/src/binder.rs | 1 + compiler/rustc_type_ir/src/predicate.rs | 12 +++++++ 13 files changed, 109 insertions(+) diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 1eb099acce70a..f1b2ab9b37f6e 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -255,10 +255,12 @@ impl<'tcx> AdtDef<'tcx> { } impl<'tcx> rustc_type_ir::inherent::AdtDef> for AdtDef<'tcx> { + #[inline] fn def_id(self) -> DefId { self.did() } + #[inline] fn is_struct(self) -> bool { self.is_struct() } @@ -273,14 +275,17 @@ impl<'tcx> rustc_type_ir::inherent::AdtDef> for AdtDef<'tcx> { Some(interner.type_of(self.non_enum_variant().tail_opt()?.did)) } + #[inline] fn is_phantom_data(self) -> bool { self.is_phantom_data() } + #[inline] fn is_manually_drop(self) -> bool { self.is_manually_drop() } + #[inline] fn field_representing_type_info( self, tcx: TyCtxt<'tcx>, @@ -298,6 +303,7 @@ impl<'tcx> rustc_type_ir::inherent::AdtDef> for AdtDef<'tcx> { ) } + #[inline] fn sizedness_constraint( self, tcx: TyCtxt<'tcx>, @@ -306,6 +312,7 @@ impl<'tcx> rustc_type_ir::inherent::AdtDef> for AdtDef<'tcx> { self.sizedness_constraint(tcx, sizedness) } + #[inline] fn is_fundamental(self) -> bool { self.is_fundamental() } diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 0a7a20496b262..f2066b9d8fdbc 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -33,6 +33,7 @@ pub struct Const<'tcx>(pub(super) Interned<'tcx, WithCachedTypeInfo rustc_type_ir::inherent::IntoKind for Const<'tcx> { type Kind = ConstKind<'tcx>; + #[inline] fn kind(self) -> ConstKind<'tcx> { self.kind() } @@ -162,14 +163,17 @@ impl<'tcx> Const<'tcx> { } impl<'tcx> rustc_type_ir::inherent::Const> for Const<'tcx> { + #[inline] fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferConst) -> Self { Const::new_infer(tcx, infer) } + #[inline] fn new_var(tcx: TyCtxt<'tcx>, vid: ty::ConstVid) -> Self { Const::new_var(tcx, vid) } + #[inline] fn new_bound( interner: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, @@ -183,22 +187,27 @@ impl<'tcx> rustc_type_ir::inherent::Const> for Const<'tcx> { Const::new_bound(tcx, debruijn, ty::BoundConst::new(var)) } + #[inline] fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: rustc_type_ir::BoundVar) -> Self { Const::new_canonical_bound(tcx, var) } + #[inline] fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderConst<'tcx>) -> Self { Const::new_placeholder(tcx, placeholder) } + #[inline] fn new_unevaluated(interner: TyCtxt<'tcx>, uv: ty::UnevaluatedConst<'tcx>) -> Self { Const::new_unevaluated(interner, uv) } + #[inline] fn new_expr(interner: TyCtxt<'tcx>, expr: ty::Expr<'tcx>) -> Self { Const::new_expr(interner, expr) } + #[inline] fn new_error(interner: TyCtxt<'tcx>, guar: ErrorGuaranteed) -> Self { Const::new_error(interner, guar) } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index bb8a342ba950b..8052941607986 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -74,10 +74,12 @@ use crate::ty::{ }; impl<'tcx> rustc_type_ir::inherent::DefId> for DefId { + #[inline] fn is_local(self) -> bool { self.is_local() } + #[inline] fn as_local(self) -> Option { self.as_local() } @@ -92,24 +94,29 @@ impl<'tcx> rustc_type_ir::inherent::Safety> for hir::Safety { hir::Safety::Unsafe } + #[inline] fn is_safe(self) -> bool { self.is_safe() } + #[inline] fn prefix_str(self) -> &'static str { self.prefix_str() } } impl<'tcx> rustc_type_ir::inherent::Features> for &'tcx rustc_feature::Features { + #[inline] fn generic_const_exprs(self) -> bool { self.generic_const_exprs() } + #[inline] fn generic_const_args(self) -> bool { self.generic_const_args() } + #[inline] fn coroutine_clone(self) -> bool { self.coroutine_clone() } diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs index 6b5733e87b14c..5291897fda907 100644 --- a/compiler/rustc_middle/src/ty/generic_args.rs +++ b/compiler/rustc_middle/src/ty/generic_args.rs @@ -41,6 +41,7 @@ pub struct GenericArg<'tcx> { impl<'tcx> rustc_type_ir::inherent::GenericArg> for GenericArg<'tcx> {} impl<'tcx> rustc_type_ir::inherent::GenericArgs> for ty::GenericArgsRef<'tcx> { + #[inline] fn rebase_onto( self, tcx: TyCtxt<'tcx>, @@ -51,24 +52,29 @@ impl<'tcx> rustc_type_ir::inherent::GenericArgs> for ty::GenericArg } #[track_caller] + #[inline] fn type_at(self, i: usize) -> Ty<'tcx> { self.type_at(i) } #[track_caller] + #[inline] fn region_at(self, i: usize) -> ty::Region<'tcx> { self.region_at(i) } #[track_caller] + #[inline] fn const_at(self, i: usize) -> ty::Const<'tcx> { self.const_at(i) } + #[inline] fn identity_for_item(tcx: TyCtxt<'tcx>, def_id: DefId) -> ty::GenericArgsRef<'tcx> { GenericArgs::identity_for_item(tcx, def_id) } + #[inline] fn extend_with_error( tcx: TyCtxt<'tcx>, def_id: DefId, @@ -130,6 +136,7 @@ impl<'tcx> rustc_type_ir::inherent::GenericArgs> for ty::GenericArg impl<'tcx> rustc_type_ir::inherent::IntoKind for GenericArg<'tcx> { type Kind = GenericArgKind<'tcx>; + #[inline] fn kind(self) -> Self::Kind { self.kind() } diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs index 87e206d5f0712..9175f6da9b36a 100644 --- a/compiler/rustc_middle/src/ty/generics.rs +++ b/compiler/rustc_middle/src/ty/generics.rs @@ -129,6 +129,7 @@ pub struct Generics { } impl<'tcx> rustc_type_ir::inherent::GenericsOf> for &'tcx Generics { + #[inline] fn count(&self) -> usize { self.parent_count + self.own_params.len() } diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index ed5a48b094f24..7cf3a5e19c6f3 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -144,10 +144,12 @@ impl<'a, H, T: Copy> rustc_type_ir::inherent::SliceLike for &'a RawList { type IntoIter = iter::Copied<<&'a [T] as IntoIterator>::IntoIter>; + #[inline] fn iter(self) -> Self::IntoIter { (*self).iter() } + #[inline] fn as_slice(&self) -> &[Self::Item] { (*self).as_slice() } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index f68526724135c..37fc392cfec75 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -480,6 +480,7 @@ pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo>>); impl<'tcx> rustc_type_ir::inherent::IntoKind for Ty<'tcx> { type Kind = TyKind<'tcx>; + #[inline] fn kind(self) -> TyKind<'tcx> { *self.kind() } @@ -520,6 +521,7 @@ impl<'tcx> rustc_type_ir::inherent::Term> for Term<'tcx> {} impl<'tcx> rustc_type_ir::inherent::IntoKind for Term<'tcx> { type Kind = TermKind<'tcx>; + #[inline] fn kind(self) -> Self::Kind { self.kind() } @@ -947,10 +949,12 @@ impl<'tcx> DefinitionSiteHiddenType<'tcx> { pub type Clauses<'tcx> = &'tcx ListWithCachedTypeInfo>; impl<'tcx> rustc_type_ir::Flags for Clauses<'tcx> { + #[inline] fn flags(&self) -> TypeFlags { (**self).flags() } + #[inline] fn outer_exclusive_binder(&self) -> DebruijnIndex { (**self).outer_exclusive_binder() } @@ -973,6 +977,7 @@ pub struct ParamEnv<'tcx> { } impl<'tcx> rustc_type_ir::inherent::ParamEnv> for ParamEnv<'tcx> { + #[inline] fn caller_bounds(self) -> impl inherent::SliceLike> { self.caller_bounds() } diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs index d776bd4185a3d..64a800ce6015f 100644 --- a/compiler/rustc_middle/src/ty/predicate.rs +++ b/compiler/rustc_middle/src/ty/predicate.rs @@ -47,6 +47,7 @@ pub struct Predicate<'tcx>( ); impl<'tcx> rustc_type_ir::inherent::Predicate> for Predicate<'tcx> { + #[inline] fn as_clause(self) -> Option> { self.as_clause() } @@ -55,6 +56,7 @@ impl<'tcx> rustc_type_ir::inherent::Predicate> for Predicate<'tcx> impl<'tcx> rustc_type_ir::inherent::IntoKind for Predicate<'tcx> { type Kind = ty::Binder<'tcx, ty::PredicateKind<'tcx>>; + #[inline] fn kind(self) -> Self::Kind { self.kind() } @@ -138,10 +140,12 @@ pub struct Clause<'tcx>( ); impl<'tcx> rustc_type_ir::inherent::Clause> for Clause<'tcx> { + #[inline] fn as_predicate(self) -> Predicate<'tcx> { self.as_predicate() } + #[inline] fn instantiate_supertrait(self, tcx: TyCtxt<'tcx>, trait_ref: ty::PolyTraitRef<'tcx>) -> Self { self.instantiate_supertrait(tcx, trait_ref) } @@ -150,6 +154,7 @@ impl<'tcx> rustc_type_ir::inherent::Clause> for Clause<'tcx> { impl<'tcx> rustc_type_ir::inherent::IntoKind for Clause<'tcx> { type Kind = ty::Binder<'tcx, ClauseKind<'tcx>>; + #[inline] fn kind(self) -> Self::Kind { self.kind() } diff --git a/compiler/rustc_middle/src/ty/region.rs b/compiler/rustc_middle/src/ty/region.rs index bd7964a9fafcd..38bb6108ed6a4 100644 --- a/compiler/rustc_middle/src/ty/region.rs +++ b/compiler/rustc_middle/src/ty/region.rs @@ -25,6 +25,7 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Region<'tcx> { } impl<'tcx> rustc_type_ir::Flags for Region<'tcx> { + #[inline] fn flags(&self) -> TypeFlags { self.type_flags() } @@ -158,6 +159,7 @@ impl<'tcx> Region<'tcx> { } impl<'tcx> rustc_type_ir::inherent::Region> for Region<'tcx> { + #[inline] fn new_bound( interner: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, @@ -166,14 +168,17 @@ impl<'tcx> rustc_type_ir::inherent::Region> for Region<'tcx> { Region::new_bound(interner, debruijn, var) } + #[inline] fn new_anon_bound(tcx: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self { Region::new_bound(tcx, debruijn, ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon }) } + #[inline] fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: rustc_type_ir::BoundVar) -> Self { Region::new_canonical_bound(tcx, var) } + #[inline] fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderRegion<'tcx>) -> Self { Region::new_placeholder(tcx, placeholder) } diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 20094f7a8677d..b01e90ab20764 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -317,6 +317,7 @@ impl<'tcx> TypeVisitable> for Pattern<'tcx> { } impl<'tcx> TypeFoldable> for Ty<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -324,12 +325,14 @@ impl<'tcx> TypeFoldable> for Ty<'tcx> { folder.try_fold_ty(self) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_ty(self) } } impl<'tcx> TypeVisitable> for Ty<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_ty(*self) } @@ -476,6 +479,7 @@ impl<'tcx> TypeSuperVisitable> for Ty<'tcx> { } impl<'tcx> TypeFoldable> for ty::Region<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -483,18 +487,21 @@ impl<'tcx> TypeFoldable> for ty::Region<'tcx> { folder.try_fold_region(self) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_region(self) } } impl<'tcx> TypeVisitable> for ty::Region<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_region(*self) } } impl<'tcx> TypeFoldable> for ty::Predicate<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -502,6 +509,7 @@ impl<'tcx> TypeFoldable> for ty::Predicate<'tcx> { folder.try_fold_predicate(self) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_predicate(self) } @@ -524,6 +532,7 @@ impl<'tcx> TypeFoldable> for ty::Clause<'tcx> { } impl<'tcx> TypeFoldable> for ty::Clauses<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -531,12 +540,14 @@ impl<'tcx> TypeFoldable> for ty::Clauses<'tcx> { folder.try_fold_clauses(self) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_clauses(self) } } impl<'tcx> TypeVisitable> for ty::Predicate<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_predicate(*self) } @@ -579,6 +590,7 @@ impl<'tcx> TypeSuperVisitable> for ty::Predicate<'tcx> { } impl<'tcx> TypeVisitable> for ty::Clauses<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_clauses(self) } @@ -604,6 +616,7 @@ impl<'tcx> TypeSuperFoldable> for ty::Clauses<'tcx> { } impl<'tcx> TypeFoldable> for ty::Const<'tcx> { + #[inline] fn try_fold_with>>( self, folder: &mut F, @@ -611,12 +624,14 @@ impl<'tcx> TypeFoldable> for ty::Const<'tcx> { folder.try_fold_const(self) } + #[inline] fn fold_with>>(self, folder: &mut F) -> Self { folder.fold_const(self) } } impl<'tcx> TypeVisitable> for ty::Const<'tcx> { + #[inline] fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_const(*self) } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index fd95d58e9f3ad..c374ab39a339f 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -935,22 +935,27 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { tcx.types.u8 } + #[inline] fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferTy) -> Self { Ty::new_infer(tcx, infer) } + #[inline] fn new_var(tcx: TyCtxt<'tcx>, vid: ty::TyVid) -> Self { Ty::new_var(tcx, vid) } + #[inline] fn new_param(tcx: TyCtxt<'tcx>, param: ty::ParamTy) -> Self { Ty::new_param(tcx, param.index, param.name) } + #[inline] fn new_placeholder(tcx: TyCtxt<'tcx>, placeholder: ty::PlaceholderType<'tcx>) -> Self { Ty::new_placeholder(tcx, placeholder) } + #[inline] fn new_bound( interner: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, @@ -959,22 +964,27 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_bound(interner, debruijn, var) } + #[inline] fn new_anon_bound(tcx: TyCtxt<'tcx>, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self { Ty::new_bound(tcx, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon }) } + #[inline] fn new_canonical_bound(tcx: TyCtxt<'tcx>, var: ty::BoundVar) -> Self { Ty::new_canonical_bound(tcx, var) } + #[inline] fn new_alias(interner: TyCtxt<'tcx>, alias_ty: ty::AliasTy<'tcx>) -> Self { Ty::new_alias(interner, alias_ty) } + #[inline] fn new_error(interner: TyCtxt<'tcx>, guar: ErrorGuaranteed) -> Self { Ty::new_error(interner, guar) } + #[inline] fn new_adt( interner: TyCtxt<'tcx>, adt_def: ty::AdtDef<'tcx>, @@ -983,10 +993,12 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_adt(interner, adt_def, args) } + #[inline] fn new_foreign(interner: TyCtxt<'tcx>, def_id: DefId) -> Self { Ty::new_foreign(interner, def_id) } + #[inline] fn new_dynamic( interner: TyCtxt<'tcx>, preds: &'tcx List>, @@ -995,6 +1007,7 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_dynamic(interner, preds, region) } + #[inline] fn new_coroutine( interner: TyCtxt<'tcx>, def_id: DefId, @@ -1003,6 +1016,7 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_coroutine(interner, def_id, args) } + #[inline] fn new_coroutine_closure( interner: TyCtxt<'tcx>, def_id: DefId, @@ -1011,10 +1025,12 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_coroutine_closure(interner, def_id, args) } + #[inline] fn new_closure(interner: TyCtxt<'tcx>, def_id: DefId, args: ty::GenericArgsRef<'tcx>) -> Self { Ty::new_closure(interner, def_id, args) } + #[inline] fn new_coroutine_witness( interner: TyCtxt<'tcx>, def_id: DefId, @@ -1023,6 +1039,7 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_coroutine_witness(interner, def_id, args) } + #[inline] fn new_coroutine_witness_for_coroutine( interner: TyCtxt<'tcx>, def_id: DefId, @@ -1031,10 +1048,12 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_coroutine_witness_for_coroutine(interner, def_id, coroutine_args) } + #[inline] fn new_ptr(interner: TyCtxt<'tcx>, ty: Self, mutbl: hir::Mutability) -> Self { Ty::new_ptr(interner, ty, mutbl) } + #[inline] fn new_ref( interner: TyCtxt<'tcx>, region: ty::Region<'tcx>, @@ -1044,18 +1063,22 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_ref(interner, region, ty, mutbl) } + #[inline] fn new_array_with_const_len(interner: TyCtxt<'tcx>, ty: Self, len: ty::Const<'tcx>) -> Self { Ty::new_array_with_const_len(interner, ty, len) } + #[inline] fn new_slice(interner: TyCtxt<'tcx>, ty: Self) -> Self { Ty::new_slice(interner, ty) } + #[inline] fn new_tup(interner: TyCtxt<'tcx>, tys: &[Ty<'tcx>]) -> Self { Ty::new_tup(interner, tys) } + #[inline] fn new_tup_from_iter(interner: TyCtxt<'tcx>, iter: It) -> T::Output where It: Iterator, @@ -1064,18 +1087,22 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_tup_from_iter(interner, iter) } + #[inline] fn tuple_fields(self) -> &'tcx ty::List> { self.tuple_fields() } + #[inline] fn to_opt_closure_kind(self) -> Option { self.to_opt_closure_kind() } + #[inline] fn from_closure_kind(interner: TyCtxt<'tcx>, kind: ty::ClosureKind) -> Self { Ty::from_closure_kind(interner, kind) } + #[inline] fn from_coroutine_closure_kind( interner: TyCtxt<'tcx>, kind: rustc_type_ir::ClosureKind, @@ -1083,18 +1110,22 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::from_coroutine_closure_kind(interner, kind) } + #[inline] fn new_fn_def(interner: TyCtxt<'tcx>, def_id: DefId, args: ty::GenericArgsRef<'tcx>) -> Self { Ty::new_fn_def(interner, def_id, args) } + #[inline] fn new_fn_ptr(interner: TyCtxt<'tcx>, sig: ty::Binder<'tcx, ty::FnSig<'tcx>>) -> Self { Ty::new_fn_ptr(interner, sig) } + #[inline] fn new_pat(interner: TyCtxt<'tcx>, ty: Self, pat: ty::Pattern<'tcx>) -> Self { Ty::new_pat(interner, ty, pat) } + #[inline] fn new_unsafe_binder(interner: TyCtxt<'tcx>, ty: ty::Binder<'tcx, Ty<'tcx>>) -> Self { Ty::new_unsafe_binder(interner, ty) } @@ -1107,10 +1138,12 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { interner.types.usize } + #[inline] fn discriminant_ty(self, interner: TyCtxt<'tcx>) -> Ty<'tcx> { self.discriminant_ty(interner) } + #[inline] fn has_unsafe_fields(self) -> bool { Ty::has_unsafe_fields(self) } diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs index ab9428571e964..6a52f892153cb 100644 --- a/compiler/rustc_type_ir/src/binder.rs +++ b/compiler/rustc_type_ir/src/binder.rs @@ -232,6 +232,7 @@ impl Binder { /// don't actually track bound vars. However, semantically, it is different /// because bound vars aren't allowed to change here, whereas they are /// in `bind`. This may be (debug) asserted in the future. + #[inline] pub fn rebind(&self, value: U) -> Binder where U: TypeVisitable, diff --git a/compiler/rustc_type_ir/src/predicate.rs b/compiler/rustc_type_ir/src/predicate.rs index fc62348a35d1b..7216a99fd47d1 100644 --- a/compiler/rustc_type_ir/src/predicate.rs +++ b/compiler/rustc_type_ir/src/predicate.rs @@ -185,6 +185,7 @@ impl ty::Binder> { self.map_bound_ref(|tr| tr.self_ty()) } + #[inline] pub fn def_id(&self) -> I::TraitId { self.skip_binder().def_id } @@ -228,6 +229,7 @@ impl TraitPredicate { self.trait_ref.def_id } + #[inline] pub fn self_ty(self) -> I::Ty { self.trait_ref.self_ty() } @@ -454,6 +456,7 @@ impl ExistentialTraitRef { } impl ty::Binder> { + #[inline] pub fn def_id(&self) -> I::TraitId { self.skip_binder().def_id } @@ -552,6 +555,7 @@ impl ty::Binder> { self.map_bound(|p| p.with_self_ty(cx, self_ty)) } + #[inline] pub fn item_def_id(&self) -> I::TraitAssocTermId { self.skip_binder().def_id } @@ -914,6 +918,7 @@ pub struct ProjectionPredicate { impl Eq for ProjectionPredicate {} impl ProjectionPredicate { + #[inline] pub fn self_ty(self) -> I::Ty { self.projection_term.self_ty() } @@ -925,10 +930,12 @@ impl ProjectionPredicate { } } + #[inline] pub fn trait_def_id(self, interner: I) -> I::TraitId { self.projection_term.trait_def_id(interner) } + #[inline] pub fn def_id(self) -> I::TraitAssocTermId { self.projection_term.expect_projection_def_id() } @@ -941,6 +948,7 @@ impl ty::Binder> { self.skip_binder().projection_term.trait_def_id(cx) } + #[inline] pub fn term(&self) -> ty::Binder { self.map_bound(|predicate| predicate.term) } @@ -978,6 +986,7 @@ pub struct NormalizesTo { impl Eq for NormalizesTo {} impl NormalizesTo { + #[inline] pub fn self_ty(self) -> I::Ty { self.alias.self_ty() } @@ -986,10 +995,12 @@ impl NormalizesTo { Self { alias: self.alias.with_replaced_self_ty(interner, self_ty), ..self } } + #[inline] pub fn trait_def_id(self, interner: I) -> I::TraitId { self.alias.trait_def_id(interner) } + #[inline] pub fn def_id(self) -> I::DefId { self.alias.def_id() } @@ -1015,6 +1026,7 @@ pub struct HostEffectPredicate { impl Eq for HostEffectPredicate {} impl HostEffectPredicate { + #[inline] pub fn self_ty(self) -> I::Ty { self.trait_ref.self_ty() } From 54ef705e099b96d760d386a5fa46e677da3565d4 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Fri, 8 May 2026 19:40:04 +0000 Subject: [PATCH 4/5] Add `inline` to pure field-access accessors These accessors have pure field/index accesses -- no calls. The auto-inline heuristic from Rust PR 116505 should already cover these. Let's annotate to confirm that in profiling. Maybe or maybe not we'd want to add these anyway as documentation of intent and a safety net against later changes that would cause the heuristic to fail. --- compiler/rustc_ast_ir/src/visit.rs | 8 ++++++++ compiler/rustc_middle/src/ty/consts.rs | 2 ++ compiler/rustc_middle/src/ty/consts/kind.rs | 1 + compiler/rustc_middle/src/ty/consts/valtree.rs | 3 +++ compiler/rustc_middle/src/ty/context.rs | 3 +++ compiler/rustc_middle/src/ty/mod.rs | 2 ++ compiler/rustc_middle/src/ty/pattern.rs | 1 + compiler/rustc_middle/src/ty/predicate.rs | 2 ++ compiler/rustc_middle/src/ty/region.rs | 3 +++ compiler/rustc_middle/src/ty/sty.rs | 7 +++++++ compiler/rustc_type_ir/src/binder.rs | 8 ++++++++ compiler/rustc_type_ir/src/predicate.rs | 2 ++ 12 files changed, 42 insertions(+) diff --git a/compiler/rustc_ast_ir/src/visit.rs b/compiler/rustc_ast_ir/src/visit.rs index 8315c080dfa86..771c561cbc2f4 100644 --- a/compiler/rustc_ast_ir/src/visit.rs +++ b/compiler/rustc_ast_ir/src/visit.rs @@ -16,9 +16,13 @@ impl VisitorResult for () { #[cfg(not(feature = "nightly"))] type Residual = core::convert::Infallible; + #[inline] fn output() -> Self {} + #[inline] fn from_residual(_: Self::Residual) -> Self {} + #[inline] fn from_branch(_: ControlFlow) -> Self {} + #[inline] fn branch(self) -> ControlFlow { ControlFlow::Continue(()) } @@ -27,15 +31,19 @@ impl VisitorResult for () { impl VisitorResult for ControlFlow { type Residual = T; + #[inline] fn output() -> Self { ControlFlow::Continue(()) } + #[inline] fn from_residual(residual: Self::Residual) -> Self { ControlFlow::Break(residual) } + #[inline] fn from_branch(b: Self) -> Self { b } + #[inline] fn branch(self) -> Self { self } diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index f2066b9d8fdbc..34f9e11f6be07 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -40,10 +40,12 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Const<'tcx> { } impl<'tcx> rustc_type_ir::Flags for Const<'tcx> { + #[inline] fn flags(&self) -> TypeFlags { self.0.flags } + #[inline] fn outer_exclusive_binder(&self) -> rustc_type_ir::DebruijnIndex { self.0.outer_exclusive_binder } diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs index a59dd3c488a22..44f3e78227280 100644 --- a/compiler/rustc_middle/src/ty/consts/kind.rs +++ b/compiler/rustc_middle/src/ty/consts/kind.rs @@ -23,6 +23,7 @@ pub struct Expr<'tcx> { } impl<'tcx> rustc_type_ir::inherent::ExprConst> for Expr<'tcx> { + #[inline] fn args(self) -> ty::GenericArgsRef<'tcx> { self.args } diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs index 91f689dcb6d8b..bd80c2ba8cbff 100644 --- a/compiler/rustc_middle/src/ty/consts/valtree.rs +++ b/compiler/rustc_middle/src/ty/consts/valtree.rs @@ -80,6 +80,7 @@ impl fmt::Debug for ValTree<'_> { impl<'tcx> rustc_type_ir::inherent::IntoKind for ty::ValTree<'tcx> { type Kind = ty::ValTreeKind>; + #[inline] fn kind(self) -> Self::Kind { *self.0 } @@ -226,10 +227,12 @@ impl<'tcx> Value<'tcx> { } impl<'tcx> rustc_type_ir::inherent::ValueConst> for Value<'tcx> { + #[inline] fn ty(self) -> Ty<'tcx> { self.ty } + #[inline] fn valtree(self) -> ValTree<'tcx> { self.valtree } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 8052941607986..61c0bb04429bc 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -86,10 +86,12 @@ impl<'tcx> rustc_type_ir::inherent::DefId> for DefId { } impl<'tcx> rustc_type_ir::inherent::Safety> for hir::Safety { + #[inline] fn safe() -> Self { hir::Safety::Safe } + #[inline] fn unsafe_mode() -> Self { hir::Safety::Unsafe } @@ -131,6 +133,7 @@ impl<'tcx> rustc_type_ir::inherent::Features> for &'tcx rustc_featu } impl<'tcx> rustc_type_ir::inherent::Span> for Span { + #[inline] fn dummy() -> Self { DUMMY_SP } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 37fc392cfec75..44a32bdf344be 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -487,10 +487,12 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Ty<'tcx> { } impl<'tcx> rustc_type_ir::Flags for Ty<'tcx> { + #[inline] fn flags(&self) -> TypeFlags { self.0.flags } + #[inline] fn outer_exclusive_binder(&self) -> DebruijnIndex { self.0.outer_exclusive_binder } diff --git a/compiler/rustc_middle/src/ty/pattern.rs b/compiler/rustc_middle/src/ty/pattern.rs index 2d73124265fc2..7ea54e002d3ca 100644 --- a/compiler/rustc_middle/src/ty/pattern.rs +++ b/compiler/rustc_middle/src/ty/pattern.rs @@ -115,6 +115,7 @@ impl<'tcx> IrPrint> for TyCtxt<'tcx> { impl<'tcx> rustc_type_ir::inherent::IntoKind for Pattern<'tcx> { type Kind = PatternKind<'tcx>; + #[inline] fn kind(self) -> Self::Kind { *self } diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs index 64a800ce6015f..09ca9fa6e9106 100644 --- a/compiler/rustc_middle/src/ty/predicate.rs +++ b/compiler/rustc_middle/src/ty/predicate.rs @@ -63,10 +63,12 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Predicate<'tcx> { } impl<'tcx> rustc_type_ir::Flags for Predicate<'tcx> { + #[inline] fn flags(&self) -> TypeFlags { self.0.flags } + #[inline] fn outer_exclusive_binder(&self) -> ty::DebruijnIndex { self.0.outer_exclusive_binder } diff --git a/compiler/rustc_middle/src/ty/region.rs b/compiler/rustc_middle/src/ty/region.rs index 38bb6108ed6a4..55d0156f7d4d8 100644 --- a/compiler/rustc_middle/src/ty/region.rs +++ b/compiler/rustc_middle/src/ty/region.rs @@ -19,6 +19,7 @@ pub struct Region<'tcx>(pub Interned<'tcx, RegionKind<'tcx>>); impl<'tcx> rustc_type_ir::inherent::IntoKind for Region<'tcx> { type Kind = RegionKind<'tcx>; + #[inline] fn kind(self) -> RegionKind<'tcx> { *self.0.0 } @@ -183,6 +184,7 @@ impl<'tcx> rustc_type_ir::inherent::Region> for Region<'tcx> { Region::new_placeholder(tcx, placeholder) } + #[inline] fn new_static(tcx: TyCtxt<'tcx>) -> Self { tcx.lifetimes.re_static } @@ -368,6 +370,7 @@ impl EarlyParamRegion { } impl rustc_type_ir::inherent::ParamLike for EarlyParamRegion { + #[inline] fn index(self) -> u32 { self.index } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index c374ab39a339f..d2af34aea8689 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -285,6 +285,7 @@ pub struct ParamTy { } impl rustc_type_ir::inherent::ParamLike for ParamTy { + #[inline] fn index(self) -> u32 { self.index } @@ -319,6 +320,7 @@ pub struct ParamConst { } impl rustc_type_ir::inherent::ParamLike for ParamConst { + #[inline] fn index(self) -> u32 { self.index } @@ -927,10 +929,12 @@ impl<'tcx> Ty<'tcx> { } impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { + #[inline] fn new_bool(tcx: TyCtxt<'tcx>) -> Self { tcx.types.bool } + #[inline] fn new_u8(tcx: TyCtxt<'tcx>) -> Self { tcx.types.u8 } @@ -1130,10 +1134,12 @@ impl<'tcx> rustc_type_ir::inherent::Ty> for Ty<'tcx> { Ty::new_unsafe_binder(interner, ty) } + #[inline] fn new_unit(interner: TyCtxt<'tcx>) -> Self { interner.types.unit } + #[inline] fn new_usize(interner: TyCtxt<'tcx>) -> Self { interner.types.usize } @@ -2191,6 +2197,7 @@ impl<'tcx> rustc_type_ir::inherent::Tys> for &'tcx ty::List rustc_type_ir::inherent::Symbol> for Symbol { + #[inline] fn is_kw_underscore_lifetime(self) -> bool { self == kw::UnderscoreLifetime } diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs index 6a52f892153cb..96db7cbf85938 100644 --- a/compiler/rustc_type_ir/src/binder.rs +++ b/compiler/rustc_type_ir/src/binder.rs @@ -168,18 +168,22 @@ impl Binder { /// /// See existing uses of `.skip_binder()` in `rustc_trait_selection::traits::select` /// or `rustc_next_trait_solver` for examples. + #[inline] pub fn skip_binder(self) -> T { self.value } + #[inline] pub fn bound_vars(&self) -> I::BoundVarKinds { self.bound_vars } + #[inline] pub fn as_ref(&self) -> Binder { Binder { value: &self.value, bound_vars: self.bound_vars } } + #[inline] pub fn as_deref(&self) -> Binder where T: Deref, @@ -394,10 +398,12 @@ generate!( ); impl EarlyBinder { + #[inline] pub fn bind(value: T) -> EarlyBinder { EarlyBinder { value, _tcx: PhantomData } } + #[inline] pub fn as_ref(&self) -> EarlyBinder { EarlyBinder { value: &self.value, _tcx: PhantomData } } @@ -428,6 +434,7 @@ impl EarlyBinder { Ok(EarlyBinder { value, _tcx: PhantomData }) } + #[inline] pub fn rebind(&self, value: U) -> EarlyBinder { EarlyBinder { value, _tcx: PhantomData } } @@ -448,6 +455,7 @@ impl EarlyBinder { /// /// See also [`Binder::skip_binder`](Binder::skip_binder), which is /// the analogous operation on [`Binder`]. + #[inline] pub fn skip_binder(self) -> T { self.value } diff --git a/compiler/rustc_type_ir/src/predicate.rs b/compiler/rustc_type_ir/src/predicate.rs index 7216a99fd47d1..8d5bcd502d509 100644 --- a/compiler/rustc_type_ir/src/predicate.rs +++ b/compiler/rustc_type_ir/src/predicate.rs @@ -225,6 +225,7 @@ impl TraitPredicate { } } + #[inline] pub fn def_id(self) -> I::TraitId { self.trait_ref.def_id } @@ -1035,6 +1036,7 @@ impl HostEffectPredicate { Self { trait_ref: self.trait_ref.with_replaced_self_ty(interner, self_ty), ..self } } + #[inline] pub fn def_id(self) -> I::TraitId { self.trait_ref.def_id } From e31b73109fe84d03f7e17f2f3f069d902d1fc173 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Sat, 9 May 2026 08:01:57 +0000 Subject: [PATCH 5/5] Outline cold body of `opt_normalize_projection_term` --- .../src/traits/project.rs | 60 ++++++++++++++----- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index e2458a2392f5f..9d42dc3fd259c 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -315,6 +315,49 @@ pub(super) fn opt_normalize_projection_term<'a, 'b, 'tcx>( // bounds. It might be the case that we want two distinct caches, // or else another kind of cache entry. let cache_entry = infcx.inner.borrow_mut().projection_cache().try_start(cache_key); + match cache_entry { + // This is the hottest path in this function. + // + // If we find the value in the cache, then return it along with the + // obligations that went along with it. Note that, when using a + // fulfillment context, these obligations could in principle be + // ignored: they have already been registered when the cache entry + // was created (and hence the new ones will quickly be discarded as + // duplicated). But when doing trait evaluation this is not the + // case, and dropping the trait evaluations can causes ICEs (e.g., + // #43132). + Err(ProjectionCacheEntry::NormalizedTerm { ty, complete: _ }) => { + debug!(?ty, "found normalized ty"); + obligations.extend(ty.obligations); + Ok(Some(ty.value)) + } + cache_entry => opt_normalize_projection_term_slow( + selcx, + param_env, + projection_term, + cause, + depth, + obligations, + cache_entry, + cache_key, + ), + } +} + +/// Cold-path body of [`opt_normalize_projection_term`]. +#[cold] +#[inline(never)] +fn opt_normalize_projection_term_slow<'a, 'b, 'tcx>( + selcx: &'a mut SelectionContext<'b, 'tcx>, + param_env: ty::ParamEnv<'tcx>, + projection_term: ty::AliasTerm<'tcx>, + cause: ObligationCause<'tcx>, + depth: usize, + obligations: &mut PredicateObligations<'tcx>, + cache_entry: Result<(), ProjectionCacheEntry<'tcx>>, + cache_key: ProjectionCacheKey<'tcx>, +) -> Result>, InProgress> { + let infcx = selcx.infcx; match cache_entry { Ok(()) => debug!("no cache"), Err(ProjectionCacheEntry::Ambiguous) => { @@ -345,21 +388,8 @@ pub(super) fn opt_normalize_projection_term<'a, 'b, 'tcx>( debug!("recur cache"); return Err(InProgress); } - Err(ProjectionCacheEntry::NormalizedTerm { ty, complete: _ }) => { - // This is the hottest path in this function. - // - // If we find the value in the cache, then return it along - // with the obligations that went along with it. Note - // that, when using a fulfillment context, these - // obligations could in principle be ignored: they have - // already been registered when the cache entry was - // created (and hence the new ones will quickly be - // discarded as duplicated). But when doing trait - // evaluation this is not the case, and dropping the trait - // evaluations can causes ICEs (e.g., #43132). - debug!(?ty, "found normalized ty"); - obligations.extend(ty.obligations); - return Ok(Some(ty.value)); + Err(ProjectionCacheEntry::NormalizedTerm { .. }) => { + unreachable!("NormalizedTerm cache hit should be handled by the hot dispatcher"); } Err(ProjectionCacheEntry::Error) => { debug!("opt_normalize_projection_type: found error");