From 378f1af5fdef6d72c2f50780c8e20fc769cc723f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:00:59 +0200 Subject: [PATCH 01/10] move overflow check --- .../rustc_trait_selection/src/solve/fulfill.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4b80d22d6bb49..4554bcf38ea8e 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -209,12 +209,6 @@ where loop { let mut any_changed = false; for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { - if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { - self.obligations.on_fulfillment_overflow(infcx); - // Only return true errors that we have accumulated while processing. - return errors; - } - let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); if !delegate.disable_trait_solver_fast_paths() @@ -261,7 +255,14 @@ where // approximation and should only result in fulfillment overflow in // pathological cases. obligation.recursion_depth += 1; - any_changed = true; + + if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) { + self.obligations.on_fulfillment_overflow(infcx); + // Only return true errors that we have accumulated while processing. + return errors; + } else { + any_changed = true; + } } match certainty { From 88ee7e462fbf263b3480eb13bf831f312e47431d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 11:30:03 +0200 Subject: [PATCH 02/10] move fast path into evaluate_goal --- .../src/solve/eval_ctxt/mod.rs | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 3a98abf06a342..957b5c21f67b1 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -592,6 +592,15 @@ where )); } + if !self.delegate.disable_trait_solver_fast_paths() + && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) + { + return Ok(( + NestedNormalizationGoals::empty(), + GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, + )); + } + self.evaluate_goal_cold(source, goal, increase_depth_for_nested) } @@ -990,20 +999,6 @@ where PredicateKind::NormalizesTo(_) )); - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - match certainty { - Certainty::Yes => {} - Certainty::Maybe { .. } => { - self.nested_goals.push((source, goal, None)); - unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty)); - } - } - continue; - } - let GoalEvaluation { goal, certainty, has_changed, stalled_on } = self.evaluate_goal(source, goal, stalled_on)?; if has_changed == HasChanged::Yes { From 238614538d6cba4e407920a4c7abe11c12719b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 13:16:33 +0200 Subject: [PATCH 03/10] remove fastpath from fullfilment loop --- .../src/solve/fulfill.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 4554bcf38ea8e..0944da1f93dfe 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,7 +7,6 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; -use rustc_next_trait_solver::delegate::SolverDelegate as _; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -211,24 +210,6 @@ where for (mut obligation, stalled_on) in self.obligations.drain_pending(|_, _| true) { let goal = obligation.as_goal(); let delegate = <&SolverDelegate<'tcx>>::from(infcx); - if !delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = - delegate.compute_goal_fast_path(goal, obligation.cause.span) - { - match certainty { - // This fast path doesn't depend on region identity so it doesn't - // matter if the goal contains inference variables or not, so we - // don't need to call `push_hir_typeck_potentially_region_dependent_goal` - // here. - // - // Only goals proven via the trait solver should be region dependent. - Certainty::Yes => {} - Certainty::Maybe(_) => { - self.obligations.register(obligation, None); - } - } - continue; - } let result = delegate.evaluate_root_goal(goal, obligation.cause.span, stalled_on); self.inspect_evaluated_obligation(infcx, &obligation, &result); From d54b2abaa48e2ccd04533b5b09386fbb8b25dbf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 18 Jun 2026 14:12:54 +0200 Subject: [PATCH 04/10] experiment with better evaluate_goal_step --- compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 957b5c21f67b1..45f65e224fe36 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -992,6 +992,9 @@ where ) -> Result, NoSolutionOrRerunNonErased> { // If this loop did not result in any progress, what's our final certainty. let mut unchanged_certainty = Some(Certainty::Yes); + // This mem::take seems super inefficient, given that we push to it again later. + // Despite that, replacing it has no effect on performance. We tried. + // (https://github.com/rust-lang/rust/pull/158126) for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) { // We never handle `NormalizesTo` as a nested goal debug_assert!(!matches!( From 15d7b5b08869735c2ceed6bef48b879cdc5ac667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 10:13:06 +0200 Subject: [PATCH 05/10] return stalled info from fast path --- compiler/rustc_middle/src/traits/solve.rs | 4 + .../rustc_next_trait_solver/src/delegate.rs | 5 +- .../src/solve/eval_ctxt/mod.rs | 200 +++++++++++------- .../rustc_next_trait_solver/src/solve/mod.rs | 20 +- .../src/solve/delegate.rs | 76 ++++--- compiler/rustc_type_ir/src/solve/mod.rs | 49 ++++- 6 files changed, 227 insertions(+), 127 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index ceecb26d242dd..c45ebb80255cc 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -16,6 +16,10 @@ pub type CanonicalInput<'tcx, P = ty::Predicate<'tcx>> = ir::solve::CanonicalInp pub type CanonicalResponse<'tcx> = ir::solve::CanonicalResponse>; pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; +pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; +pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; +pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs index 2ebb0fb851a4c..7b66667486fc9 100644 --- a/compiler/rustc_next_trait_solver/src/delegate.rs +++ b/compiler/rustc_next_trait_solver/src/delegate.rs @@ -1,7 +1,8 @@ use std::ops::Deref; use rustc_type_ir::solve::{ - Certainty, FetchEligibleAssocItemResponse, Goal, NoSolution, VisibleForLeakCheck, + Certainty, ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, NoSolution, + VisibleForLeakCheck, }; use rustc_type_ir::{self as ty, InferCtxtLike, Interner, TypeFoldable}; @@ -23,7 +24,7 @@ pub trait SolverDelegate: Deref + Sized { &self, goal: Goal::Predicate>, span: ::Span, - ) -> Option; + ) -> ComputeGoalFastPathOutcome; fn fresh_var_for_kind_with_span( &self, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 45f65e224fe36..82adee3a55c29 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,9 +10,10 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, LowerAvailableDepth, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, - NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, - RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, + AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, + FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, + QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, + SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -35,7 +36,7 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; @@ -506,13 +507,8 @@ where } // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { - num_opaques, - ref stalled_vars, - ref sub_roots, - stalled_certainty, - ref previously_succeeded_in_erased, - }) = stalled_on + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on else { return MayMakeProgress; }; @@ -529,34 +525,46 @@ where return MayMakeProgress; } - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!("we never retry stalled queries if the parent was erased") + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) + { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = + previously_succeeded_in_erased + { + match self.should_rerun_after_erased_canonicalization( + accessed_opaques, + self.typing_mode(), + &self.delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!( + "we never retry stalled queries if the parent was erased" + ) + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; } } } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } } // Otherwise, we can be sure that this stalled goal cannot make any progress @@ -564,6 +572,39 @@ where WontMakeProgress(stalled_certainty) } + /// This is a fast path optimization: + /// See the docs on [`ComputeGoalFastPathOutcome`] + pub fn compute_goal_fast_path( + &self, + goal: Goal, + ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { + if self.delegate.disable_trait_solver_fast_paths() { + return None; + } + + match self.delegate.compute_goal_fast_path(goal, self.origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }, + )), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( + NestedNormalizationGoals::empty(), + GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }, + )), + } + } + /// Recursively evaluates `goal`, returning the nested goals in case /// the nested goal is a `NormalizesTo` goal. /// @@ -592,13 +633,8 @@ where )); } - if !self.delegate.disable_trait_solver_fast_paths() - && let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { goal, certainty, has_changed: HasChanged::No, stalled_on: None }, - )); + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); } self.evaluate_goal_cold(source, goal, increase_depth_for_nested) @@ -768,41 +804,12 @@ where // that is not resolved. Only when *these* have changed is it meaningful // to recompute this goal. HasChanged::Yes => None, - HasChanged::No => { - // Remove the canonicalized universal vars, since we only care about stalled existentials. - let mut sub_roots = Vec::new(); - let mut stalled_vars = orig_values; - stalled_vars.retain(|arg| match arg.kind() { - // Lifetimes can never stall goals. - ty::GenericArgKind::Lifetime(_) => false, - ty::GenericArgKind::Type(ty) => match ty.kind() { - ty::Infer(ty::TyVar(vid)) => { - sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); - true - } - ty::Infer(_) => true, - ty::Param(_) | ty::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ty:?}"), - }, - ty::GenericArgKind::Const(ct) => match ct.kind() { - ty::ConstKind::Infer(_) => true, - ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, - _ => unreachable!("unexpected orig_value: {ct:?}"), - }, - }); - - Some(GoalStalledOn { - num_opaques: canonical_goal - .canonical - .value - .predefined_opaques_in_body - .len(), - stalled_vars, - sub_roots, - stalled_certainty: certainty, - previously_succeeded_in_erased: succeeded_in_erased, - }) - } + HasChanged::No => Some(self.build_stalled_on( + canonical_goal, + certainty, + orig_values, + succeeded_in_erased, + )), }, }; @@ -812,6 +819,45 @@ where )) } + fn build_stalled_on( + &self, + canonical_goal: CanonicalInput, + certainty: Certainty, + mut stalled_vars: Vec, + previously_succeeded_in_erased: SucceededInErased, + ) -> GoalStalledOn { + // Remove the canonicalized universal vars, since we only care about stalled existentials. + let mut sub_roots = Vec::new(); + stalled_vars.retain(|arg| match arg.kind() { + // Lifetimes can never stall goals. + ty::GenericArgKind::Lifetime(_) => false, + ty::GenericArgKind::Type(ty) => match ty.kind() { + ty::Infer(ty::TyVar(vid)) => { + sub_roots.push(self.delegate.sub_unification_table_root_var(vid)); + true + } + ty::Infer(_) => true, + ty::Param(_) | ty::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ty:?}"), + }, + ty::GenericArgKind::Const(ct) => match ct.kind() { + ty::ConstKind::Infer(_) => true, + ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) => false, + _ => unreachable!("unexpected orig_value: {ct:?}"), + }, + }); + + GoalStalledOn { + stalled_vars, + sub_roots, + stalled_certainty: certainty, + reason: GoalStalledOnReason::Other { + num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + previously_succeeded_in_erased, + }, + } + } + fn should_rerun_after_erased_canonicalization( &self, AccessedOpaques { reason: _, rerun }: AccessedOpaques, diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index fcf8ddf59fe3c..100ef083ac727 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -23,7 +23,7 @@ mod trait_goals; use derive_where::derive_where; use rustc_type_ir::inherent::*; pub use rustc_type_ir::solve::*; -use rustc_type_ir::{self as ty, Interner, TyVid}; +use rustc_type_ir::{self as ty, Interner}; use tracing::instrument; pub use self::eval_ctxt::{ @@ -424,21 +424,3 @@ pub struct GoalEvaluation { /// before rerunning it. pub stalled_on: Option>, } - -/// The conditions that must change for a goal to warrant -#[derive_where(Clone, Debug; I: Interner)] -pub struct GoalStalledOn { - pub num_opaques: usize, - pub stalled_vars: Vec, - pub sub_roots: Vec, - /// The certainty that will be returned on subsequent evaluations if this - /// goal remains stalled. - pub stalled_certainty: Certainty, - pub previously_succeeded_in_erased: SucceededInErased, -} - -#[derive_where(Clone, Debug; I: Interner)] -pub enum SucceededInErased { - Yes { accessed_opaques: AccessedOpaques }, - No, -} diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index e0877829b25f3..913cf28cd2426 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -10,12 +10,15 @@ use rustc_infer::infer::canonical::{ QueryRegionConstraint, }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; -use rustc_infer::traits::solve::{FetchEligibleAssocItemResponse, Goal}; +use rustc_infer::traits::solve::{ + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, +}; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -73,10 +76,25 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &self, goal: Goal<'tcx, ty::Predicate<'tcx>>, span: Span, - ) -> Option { + ) -> ComputeGoalFastPathOutcome<'tcx> { + use ComputeGoalFastPathOutcome as Outcome; + + fn stalled_with_args<'tcx>( + stalled_vars: Vec>, + ) -> ComputeGoalFastPathOutcome<'tcx> { + Outcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + reason: GoalStalledOnReason::FastPath, + }, + } + } + // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { - return None; + return Outcome::NoFastPath; } let pred = goal.predicate.kind(); @@ -84,22 +102,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_pred)) => { let trait_pred = pred.rebind(trait_pred); - if self.shallow_resolve(trait_pred.self_ty().skip_binder()).is_ty_var() + let self_ty = self.shallow_resolve(trait_pred.self_ty().skip_binder()); + if self_ty.is_ty_var() // We don't do this fast path when opaques are defined since we may // eventually use opaques to incompletely guide inference via ty var // self types. // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![self_ty.into()]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } Some(LangItem::Copy | LangItem::Clone) => { @@ -114,23 +133,23 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Some(Certainty::Yes); + return Outcome::TriviallyHolds; } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } else { - None + Outcome::NoFastPath } } ty::PredicateKind::DynCompatible(def_id) if self.0.tcx.is_dyn_compatible(def_id) => { - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.sub_regions( @@ -139,11 +158,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< outlives.0, ty::VisibleForLeakCheck::Yes, ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => { if outlives.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } self.0.register_type_outlives_constraint( @@ -152,48 +171,49 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< &ObligationCause::dummy_with_span(span), ); - Some(Certainty::Yes) + Outcome::TriviallyHolds } ty::PredicateKind::Subtype(ty::SubtypePredicate { a, b, .. }) | ty::PredicateKind::Coerce(ty::CoercePredicate { a, b }) => { if a.has_escaping_bound_vars() || b.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![a.into(), b.into()]) } - _ => None, + _ => Outcome::NoFastPath, } } ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, _)) => { if ct.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } - if self.shallow_resolve_const(ct).is_ct_infer() { - Some(Certainty::AMBIGUOUS) + let arg = self.shallow_resolve_const(ct); + if arg.is_ct_infer() { + stalled_with_args(vec![arg.into()]) } else { - None + Outcome::NoFastPath } } ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => { if arg.has_escaping_bound_vars() { - return None; + return Outcome::NoFastPath; } let arg = self.shallow_resolve_term(arg); if arg.is_trivially_wf(self.tcx) { - Some(Certainty::Yes) + Outcome::TriviallyHolds } else if arg.is_infer() { - Some(Certainty::AMBIGUOUS) + stalled_with_args(vec![arg.into_arg()]) } else { - None + Outcome::NoFastPath } } - _ => None, + _ => Outcome::NoFastPath, } } diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 8866c763927b3..5b07a23387ecb 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -16,7 +16,7 @@ use crate::lang_items::SolverTraitLangItem; use crate::region_constraint::RegionConstraint; use crate::search_graph::PathKind; use crate::{ - self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TypingMode, Upcast, + self as ty, Canonical, CanonicalVarValues, CantBeErased, Interner, TyVid, TypingMode, Upcast, }; pub type CanonicalInput::Predicate> = @@ -942,3 +942,50 @@ impl SizedTraitKind { }) } } + +#[derive_where(Clone, Debug; I: Interner)] +pub enum SucceededInErased { + /// This goal previously succeeded in erased mode, which based on `accessed_opaques` + /// might make us take a fast path slightly more often. + Yes { + accessed_opaques: AccessedOpaques, + }, + No, +} + +#[derive_where(Clone, Debug; I: Interner)] +pub enum GoalStalledOnReason { + /// This goal got stalled in `compute_goal_fast_path`. Usually this means + /// the goal is stalled on not that much, only one or two variables, and + /// definitely nothing to do with opaque types. So we don't store that information. + FastPath, + Other { + num_opaques: usize, + previously_succeeded_in_erased: SucceededInErased, + }, +} + +/// The conditions that must change for a goal to warrant +#[derive_where(Clone, Debug; I: Interner)] +pub struct GoalStalledOn { + pub stalled_vars: Vec, + pub sub_roots: Vec, + /// The certainty that will be returned on subsequent evaluations if this + /// goal remains stalled. + pub stalled_certainty: Certainty, + pub reason: GoalStalledOnReason, +} + +/// For some goals we can trivially answer some questions without going through +/// canonicalization. There are three options: + +#[derive(Clone, Debug)] +pub enum ComputeGoalFastPathOutcome { + /// Do not attempt the fast path. Compute as normal. + NoFastPath, + /// The goal trivially holds, immediately produce a result with [`Certainty::Yes`] + TriviallyHolds, + /// The goal is trivially stalled: we know for sure that it makes no sense to compute it right + /// now, but can return information about what its stalled on and when it can be computed for real. + TriviallyStalled { stalled_on: GoalStalledOn }, +} From 47b6070fbb816c539747e4f0c86f7b6b3f1a98e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Mon, 22 Jun 2026 14:19:47 +0200 Subject: [PATCH 06/10] move goal fast path into evaluate_goal_cold --- .../rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 82adee3a55c29..14283cd57ea1c 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -633,10 +633,6 @@ where )); } - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - self.evaluate_goal_cold(source, goal, increase_depth_for_nested) } @@ -648,6 +644,10 @@ where goal: Goal, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { + if let Some(res) = self.compute_goal_fast_path(goal) { + return Ok(res); + } + // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. From f806e9b85a6da82430b7c0d4e4744a52f1da269c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 10:25:23 +0200 Subject: [PATCH 07/10] shuffle fastpaths around to before creating an evalctxt --- .../src/solve/eval_ctxt/fast_path.rs | 136 ++++++ .../src/solve/eval_ctxt/mod.rs | 399 +++++++----------- .../src/solve/project_goals/mod.rs | 1 - 3 files changed, 281 insertions(+), 255 deletions(-) create mode 100644 compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs new file mode 100644 index 0000000000000..a6de467102bb8 --- /dev/null +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -0,0 +1,136 @@ +//! This file contains a number of standalone functions useful for taking _fast paths_ in the trait +//! solver. The exact place where we check for these fast paths changes, and matters a lot for +//! performance. Ideally we'd only check them in `evaluate_goal`, but when evaluating root goals +//! we can check them earlier and save some time creating an `EvalCtxt` in the first place. +//! +//! For debugging, fast paths can be disabled using `-Zdisable-fast-paths`. + +use rustc_type_ir::inherent::*; +use rustc_type_ir::solve::{ + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + SucceededInErased, +}; +use rustc_type_ir::{InferCtxtLike, Interner}; + +use crate::delegate::SolverDelegate; +use crate::solve::eval_ctxt::{RerunDecision, should_rerun_after_erased_canonicalization}; +use crate::solve::{GoalEvaluation, HasChanged}; + +#[derive(Debug, Clone, Copy)] +pub(super) enum RerunStalled { + WontMakeProgress(Certainty), + MayMakeProgress, +} + +/// If we have run a goal before, and it was stalled, check that any of the goal's +/// args have changed. This is a cheap way to determine that if we were to rerun this goal now, +/// it will remain stalled since it'll canonicalize the same way and evaluation is pure. +/// Therefore, we can skip this rerun +pub(super) fn rerunning_stalled_goal_may_make_progress( + delegate: &D, + stalled_on: Option<&GoalStalledOn>, +) -> RerunStalled +where + D: SolverDelegate, + I: Interner, +{ + use RerunStalled::*; + + // If fast paths are turned off, then we assume all goals can always make progress + if delegate.disable_trait_solver_fast_paths() { + return MayMakeProgress; + } + + // If the goal isn't stalled, we should definitely run it. + let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + stalled_on + else { + return MayMakeProgress; + }; + + // If any of the stalled goal's generic arguments changed, + // rerunning might make progress so we should rerun. + if stalled_vars.iter().any(|value| delegate.is_changed_arg(*value)) { + return MayMakeProgress; + } + + // If some inference took place in any of the sub roots, + // rerunning might make progress so we should rerun. + if sub_roots.iter().any(|&vid| delegate.sub_unification_table_root_var(vid) != vid) { + return MayMakeProgress; + } + + match reason { + GoalStalledOnReason::FastPath => { + // fastpath is never because of opaques, we can skip this check + } + &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + // If any opaques changed in the opaque type storage, + // rerunning might make progress so we should rerun. + if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + // Unless this goal previously succeeded in erased mode. + // If the stalled goal successfully evaluated while erasing opaque types, + // and the current state of the opaque type storage is not different in a way that is + // relevant, this stalled goal cannot make any progress and we set this variable to true. + let mut previous_erased_run_is_still_valid = false; + + if let &SucceededInErased::Yes { accessed_opaques } = previously_succeeded_in_erased + { + match should_rerun_after_erased_canonicalization( + accessed_opaques, + delegate.typing_mode_raw(), + &delegate.clone_opaque_types_lookup_table(), + ) { + RerunDecision::Yes => {} + RerunDecision::EagerlyPropagateToParent => { + unreachable!("we never retry stalled queries if the parent was erased") + } + RerunDecision::No => { + previous_erased_run_is_still_valid = true; + } + } + } + + if !previous_erased_run_is_still_valid { + return MayMakeProgress; + } + } + } + } + + // Otherwise, we can be sure that this stalled goal cannot make any progress + // and we can exit early. + WontMakeProgress(stalled_certainty) +} + +/// This is a fast path optimization: +/// See the docs on [`ComputeGoalFastPathOutcome`] +pub(super) fn compute_goal_fast_path( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + if delegate.disable_trait_solver_fast_paths() { + return None; + } + + match delegate.compute_goal_fast_path(goal, origin_span) { + ComputeGoalFastPathOutcome::NoFastPath => None, + ComputeGoalFastPathOutcome::TriviallyHolds => Some(GoalEvaluation { + goal, + certainty: Certainty::Yes, + has_changed: HasChanged::No, + stalled_on: None, + }), + ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(GoalEvaluation { + goal, + certainty: Certainty::AMBIGUOUS, + has_changed: HasChanged::No, + stalled_on: Some(stalled_on), + }), + } +} diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index 14283cd57ea1c..e762f3fc61308 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -10,10 +10,9 @@ use rustc_type_ir::relate::Relate; use rustc_type_ir::relate::solver_relating::RelateExt; use rustc_type_ir::search_graph::{CandidateHeadUsages, LowerAvailableDepth, PathKind}; use rustc_type_ir::solve::{ - AccessedOpaques, ComputeGoalFastPathOutcome, ExternalRegionConstraints, - FetchEligibleAssocItemResponse, MaybeInfo, NoSolutionOrRerunNonErased, OpaqueTypesJank, - QueryResultOrRerunNonErased, RerunCondition, RerunNonErased, RerunReason, RerunResultExt, - SmallCopyList, + AccessedOpaques, ExternalRegionConstraints, FetchEligibleAssocItemResponse, MaybeInfo, + NoSolutionOrRerunNonErased, OpaqueTypesJank, QueryResultOrRerunNonErased, RerunCondition, + RerunNonErased, RerunReason, RerunResultExt, SmallCopyList, }; use rustc_type_ir::{ self as ty, CanonicalVarValues, ClauseKind, InferCtxtLike, Interner, MayBeErased, @@ -32,6 +31,9 @@ use crate::delegate::SolverDelegate; use crate::normalize::{NormalizationFolder, NormalizationWasAmbiguous}; use crate::placeholder::BoundVarReplacer; use crate::resolve::eager_resolve_vars; +use crate::solve::eval_ctxt::fast_path::{ + RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, +}; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -41,6 +43,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; +mod fast_path; mod probe; mod solver_region_constraints; @@ -90,12 +93,6 @@ impl CurrentGoalKind { } } -#[derive(Debug)] -enum RerunDecision { - Yes, - No, - EagerlyPropagateToParent, -} pub struct EvalCtxt<'a, D, I = ::Interner> where D: SolverDelegate, @@ -227,8 +224,25 @@ where span: I::Span, stalled_on: Option>, ) -> Result, NoSolution> { + // Run fast paths *before* building an `EvalCtxt`, saving a little bit of time. + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); + } + + if let Some(res) = compute_goal_fast_path(self, goal, span) { + return Ok(res); + } + let result = EvalCtxt::enter_root(self, self.cx().recursion_limit(), span, |ecx| { - ecx.evaluate_goal(GoalSource::Misc, goal, stalled_on) + // Fast paths handled above + ecx.evaluate_goal_no_fast_paths(GoalSource::Misc, goal) }); match result { @@ -286,12 +300,6 @@ where } } -#[derive(Debug, Clone, Copy)] -enum RerunStalled { - WontMakeProgress(Certainty), - MayMakeProgress, -} - impl<'a, D, I> EvalCtxt<'a, D> where D: SolverDelegate, @@ -484,125 +492,35 @@ where goal: Goal, stalled_on: Option>, ) -> Result, NoSolutionOrRerunNonErased> { - let (normalization_nested_goals, goal_evaluation) = - self.evaluate_goal_raw(source, goal, stalled_on, LowerAvailableDepth::Yes)?; - assert!(normalization_nested_goals.is_empty()); - Ok(goal_evaluation) - } - - /// This is a fast path optimization: - /// If we have run this goal before, and it was stalled, check that any of the goal's - /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, - /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. - /// Therefore, we can skip this rerun - fn rerunning_stalled_goal_may_make_progress( - &self, - stalled_on: Option<&GoalStalledOn>, - ) -> RerunStalled { - use RerunStalled::*; - - // If fast paths are turned off, then we assume all goals can always make progress - if self.delegate.disable_trait_solver_fast_paths() { - return MayMakeProgress; - } - - // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = - stalled_on - else { - return MayMakeProgress; - }; - - // If any of the stalled goal's generic arguments changed, - // rerunning might make progress so we should rerun. - if stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value)) { - return MayMakeProgress; - } - - // If some inference took place in any of the sub roots, - // rerunning might make progress so we should rerun. - if sub_roots.iter().any(|&vid| self.delegate.sub_unification_table_root_var(vid) != vid) { - return MayMakeProgress; + if let RerunStalled::WontMakeProgress(stalled_certainty) = + rerunning_stalled_goal_may_make_progress(self.delegate, stalled_on.as_ref()) + { + return Ok(GoalEvaluation { + goal, + certainty: stalled_certainty, + has_changed: HasChanged::No, + stalled_on, + }); } - match reason { - GoalStalledOnReason::FastPath => { - // fastpath is never because of opaques, we can skip this check - } - &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { - // If any opaques changed in the opaque type storage, - // rerunning might make progress so we should rerun. - if self.delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) - { - // Unless this goal previously succeeded in erased mode. - // If the stalled goal successfully evaluated while erasing opaque types, - // and the current state of the opaque type storage is not different in a way that is - // relevant, this stalled goal cannot make any progress and we set this variable to true. - let mut previous_erased_run_is_still_valid = false; - - if let &SucceededInErased::Yes { accessed_opaques } = - previously_succeeded_in_erased - { - match self.should_rerun_after_erased_canonicalization( - accessed_opaques, - self.typing_mode(), - &self.delegate.clone_opaque_types_lookup_table(), - ) { - RerunDecision::Yes => {} - RerunDecision::EagerlyPropagateToParent => { - unreachable!( - "we never retry stalled queries if the parent was erased" - ) - } - RerunDecision::No => { - previous_erased_run_is_still_valid = true; - } - } - } - - if !previous_erased_run_is_still_valid { - return MayMakeProgress; - } - } - } + if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + return Ok(res); } - // Otherwise, we can be sure that this stalled goal cannot make any progress - // and we can exit early. - WontMakeProgress(stalled_certainty) + self.evaluate_goal_no_fast_paths(source, goal) } - /// This is a fast path optimization: - /// See the docs on [`ComputeGoalFastPathOutcome`] - pub fn compute_goal_fast_path( - &self, + // Outlining and `#[cold]` matter here because fast paths make it less likely to get here. + #[cold] + #[inline(never)] + fn evaluate_goal_no_fast_paths( + &mut self, + source: GoalSource, goal: Goal, - ) -> Option<(NestedNormalizationGoals, GoalEvaluation)> { - if self.delegate.disable_trait_solver_fast_paths() { - return None; - } - - match self.delegate.compute_goal_fast_path(goal, self.origin_span) { - ComputeGoalFastPathOutcome::NoFastPath => None, - ComputeGoalFastPathOutcome::TriviallyHolds => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::Yes, - has_changed: HasChanged::No, - stalled_on: None, - }, - )), - ComputeGoalFastPathOutcome::TriviallyStalled { stalled_on } => Some(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: Certainty::AMBIGUOUS, - has_changed: HasChanged::No, - stalled_on: Some(stalled_on), - }, - )), - } + ) -> Result, NoSolutionOrRerunNonErased> { + let (normalization_nested_goals, goal_evaluation) = + self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + Ok(goal_evaluation) } /// Recursively evaluates `goal`, returning the nested goals in case @@ -613,41 +531,11 @@ where /// should use [`EvalCtxt::evaluate_goal`] which discards that empty /// storage. pub(super) fn evaluate_goal_raw( - &mut self, - source: GoalSource, - goal: Goal, - stalled_on: Option>, - increase_depth_for_nested: LowerAvailableDepth, - ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let RerunStalled::WontMakeProgress(stalled_certainty) = - self.rerunning_stalled_goal_may_make_progress(stalled_on.as_ref()) - { - return Ok(( - NestedNormalizationGoals::empty(), - GoalEvaluation { - goal, - certainty: stalled_certainty, - has_changed: HasChanged::No, - stalled_on, - }, - )); - } - - self.evaluate_goal_cold(source, goal, increase_depth_for_nested) - } - - #[cold] - #[inline(never)] - pub(super) fn evaluate_goal_cold( &mut self, source: GoalSource, goal: Goal, increase_depth_for_nested: LowerAvailableDepth, ) -> Result<(NestedNormalizationGoals, GoalEvaluation), NoSolutionOrRerunNonErased> { - if let Some(res) = self.compute_goal_fast_path(goal) { - return Ok(res); - } - // We only care about one entry per `OpaqueTypeKey` here, // so we only canonicalize the lookup table and ignore // duplicate entries. @@ -714,7 +602,7 @@ where &mut inspect::ProofTreeBuilder::new_noop(), ); - let should_rerun = self.should_rerun_after_erased_canonicalization( + let should_rerun = should_rerun_after_erased_canonicalization( accessed_opaques, self.typing_mode(), &opaque_types, @@ -858,100 +746,6 @@ where } } - fn should_rerun_after_erased_canonicalization( - &self, - AccessedOpaques { reason: _, rerun }: AccessedOpaques, - original_typing_mode: TypingMode, - parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], - ) -> RerunDecision { - let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); - let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { - if defids.as_ref().is_empty() { - RerunDecision::No - } else if opaques - .iter() - .chain(parent_opaque_defids) - .any(|opaque| defids.as_ref().contains(&opaque)) - { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - let any_opaque_has_infer_as_hidden = || { - if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { - RerunDecision::Yes - } else { - RerunDecision::No - } - }; - - let res = match (rerun, original_typing_mode) { - // ============================= - (RerunCondition::Never, _) => RerunDecision::No, - // ============================= - (_, TypingMode::ErasedNotCoherence(MayBeErased)) => { - RerunDecision::EagerlyPropagateToParent - } - // ============================= - // In coherence, we never switch to erased mode, so we will never register anything - // in the rerun state, so we should've taken the first branch of this match - (_, TypingMode::Coherence) => unreachable!(), - // ============================= - (RerunCondition::Always, _) => RerunDecision::Yes, - // ============================= - ( - RerunCondition::OpaqueInStorage(..), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::Yes, - ( - RerunCondition::OpaqueInStorage(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - // ============================= - (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { - any_opaque_has_infer_as_hidden() - } - ( - RerunCondition::AnyOpaqueHasInferAsHidden, - TypingMode::PostBorrowck { .. } - | TypingMode::PostAnalysis - | TypingMode::Codegen - | TypingMode::PostTypeckUntilBorrowck { .. }, - ) => RerunDecision::No, - // ============================= - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), - TypingMode::PostAnalysis | TypingMode::Codegen, - ) => RerunDecision::Yes, - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, - ) => { - if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { - RerunDecision::Yes - } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { - RerunDecision::Yes - } else { - RerunDecision::No - } - } - ( - RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), - TypingMode::PostBorrowck { defined_opaque_types: opaques } - | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, - ) => opaque_in_storage(opaques, defids), - }; - - debug!( - "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" - ); - - res - } - pub(super) fn compute_goal( &mut self, goal: Goal, @@ -1779,6 +1573,103 @@ where } } +#[derive(Debug)] +enum RerunDecision { + Yes, + No, + EagerlyPropagateToParent, +} + +fn should_rerun_after_erased_canonicalization( + AccessedOpaques { reason: _, rerun }: AccessedOpaques, + original_typing_mode: TypingMode, + parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], +) -> RerunDecision { + let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); + let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { + if defids.as_ref().is_empty() { + RerunDecision::No + } else if opaques + .iter() + .chain(parent_opaque_defids) + .any(|opaque| defids.as_ref().contains(&opaque)) + { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + let any_opaque_has_infer_as_hidden = || { + if parent_opaque_types.iter().any(|(_, ty)| ty.is_ty_var()) { + RerunDecision::Yes + } else { + RerunDecision::No + } + }; + + let res = match (rerun, original_typing_mode) { + // ============================= + (RerunCondition::Never, _) => RerunDecision::No, + // ============================= + (_, TypingMode::ErasedNotCoherence(MayBeErased)) => RerunDecision::EagerlyPropagateToParent, + // ============================= + // In coherence, we never switch to erased mode, so we will never register anything + // in the rerun state, so we should've taken the first branch of this match + (_, TypingMode::Coherence) => unreachable!(), + // ============================= + (RerunCondition::Always, _) => RerunDecision::Yes, + // ============================= + (RerunCondition::OpaqueInStorage(..), TypingMode::PostAnalysis | TypingMode::Codegen) => { + RerunDecision::Yes + } + ( + RerunCondition::OpaqueInStorage(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::Typeck { defining_opaque_types_and_generators: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + // ============================= + (RerunCondition::AnyOpaqueHasInferAsHidden, TypingMode::Typeck { .. }) => { + any_opaque_has_infer_as_hidden() + } + ( + RerunCondition::AnyOpaqueHasInferAsHidden, + TypingMode::PostBorrowck { .. } + | TypingMode::PostAnalysis + | TypingMode::Codegen + | TypingMode::PostTypeckUntilBorrowck { .. }, + ) => RerunDecision::No, + // ============================= + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(_), + TypingMode::PostAnalysis | TypingMode::Codegen, + ) => RerunDecision::Yes, + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::Typeck { defining_opaque_types_and_generators: opaques }, + ) => { + if let RerunDecision::Yes = any_opaque_has_infer_as_hidden() { + RerunDecision::Yes + } else if let RerunDecision::Yes = opaque_in_storage(opaques, defids) { + RerunDecision::Yes + } else { + RerunDecision::No + } + } + ( + RerunCondition::OpaqueInStorageOrAnyOpaqueHasInferAsHidden(defids), + TypingMode::PostBorrowck { defined_opaque_types: opaques } + | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, + ) => opaque_in_storage(opaques, defids), + }; + + debug!( + "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" + ); + + res +} + /// Do not call this directly, use the `tcx` query instead. pub fn evaluate_root_goal_for_proof_tree_raw_provider< D: SolverDelegate, diff --git a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs index 52ef40daf40da..41fe4105cb745 100644 --- a/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/project_goals/mod.rs @@ -71,7 +71,6 @@ where ) = self.evaluate_goal_raw( GoalSource::TypeRelating, normalizes_to, - None, // We don't lower thr available depth for this `NormalizesTo` goal, as evaluating // it is an extra step only exists in the new solver that behaves like a function // call rather than an independent nested goal evaluation. So, decreasing the From 203832a839a3d0210a1c0e34e413d274bdf59d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 25 Jun 2026 13:26:20 +0200 Subject: [PATCH 08/10] fast path when adding goals --- .../src/solve/eval_ctxt/fast_path.rs | 17 ++++++++- .../src/solve/eval_ctxt/mod.rs | 35 ++++++++++++++++--- .../rustc_next_trait_solver/src/solve/mod.rs | 2 +- .../src/solve/fulfill.rs | 18 +++++++++- 4 files changed, 65 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index a6de467102bb8..6a7d49b36b64f 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -26,6 +26,7 @@ pub(super) enum RerunStalled { /// args have changed. This is a cheap way to determine that if we were to rerun this goal now, /// it will remain stalled since it'll canonicalize the same way and evaluation is pure. /// Therefore, we can skip this rerun +#[inline] pub(super) fn rerunning_stalled_goal_may_make_progress( delegate: &D, stalled_on: Option<&GoalStalledOn>, @@ -103,9 +104,23 @@ where WontMakeProgress(stalled_certainty) } +#[cold] +#[inline(never)] +pub(super) fn compute_goal_fast_path_cold( + delegate: &D, + goal: Goal, + origin_span: I::Span, +) -> Option> +where + D: SolverDelegate, + I: Interner, +{ + compute_goal_fast_path(delegate, goal, origin_span) +} + /// This is a fast path optimization: /// See the docs on [`ComputeGoalFastPathOutcome`] -pub(super) fn compute_goal_fast_path( +pub fn compute_goal_fast_path( delegate: &D, goal: Goal, origin_span: I::Span, diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index e762f3fc61308..ded62b4c81f92 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -34,6 +34,7 @@ use crate::resolve::eager_resolve_vars; use crate::solve::eval_ctxt::fast_path::{ RerunStalled, compute_goal_fast_path, rerunning_stalled_goal_may_make_progress, }; +use crate::solve::fast_path::compute_goal_fast_path_cold; use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ @@ -43,7 +44,7 @@ use crate::solve::{ VisibleForLeakCheck, inspect, }; -mod fast_path; +pub mod fast_path; mod probe; mod solver_region_constraints; @@ -236,7 +237,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self, goal, span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self, goal, span) + { return Ok(res); } @@ -503,7 +510,13 @@ where }); } - if let Some(res) = compute_goal_fast_path(self.delegate, goal, self.origin_span) { + if + // No need to try the fast path if stalled_on is `None`, since we already try the fast path + // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying + // the fast path twice for some goals. + stalled_on.is_some() + && let Some(res) = compute_goal_fast_path_cold(self.delegate, goal, self.origin_span) + { return Ok(res); } @@ -520,6 +533,7 @@ where ) -> Result, NoSolutionOrRerunNonErased> { let (normalization_nested_goals, goal_evaluation) = self.evaluate_goal_raw(source, goal, LowerAvailableDepth::Yes)?; + assert!(normalization_nested_goals.is_empty()); Ok(goal_evaluation) } @@ -881,7 +895,20 @@ where ty::Unnormalized::new_wip(goal.predicate), )?; self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal); - self.nested_goals.push((source, goal, None)); + + if let Some(GoalEvaluation { goal, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(self.delegate, goal, self.origin_span) + { + match certainty { + // We're done here + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.nested_goals.push((source, goal, stalled_on)); + } + } + } else { + self.nested_goals.push((source, goal, None)); + } Ok(()) } diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs index 100ef083ac727..452a87a89a7f1 100644 --- a/compiler/rustc_next_trait_solver/src/solve/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs @@ -28,7 +28,7 @@ use tracing::instrument; pub use self::eval_ctxt::{ EvalCtxt, GenerateProofTree, SolverDelegateEvalExt, - evaluate_root_goal_for_proof_tree_raw_provider, + evaluate_root_goal_for_proof_tree_raw_provider, fast_path, }; use crate::delegate::SolverDelegate; use crate::solve::assembly::Candidate; diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs index 0944da1f93dfe..4c2c92ebc5072 100644 --- a/compiler/rustc_trait_selection/src/solve/fulfill.rs +++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs @@ -7,6 +7,7 @@ use rustc_infer::traits::{ FromSolverError, PredicateObligation, PredicateObligations, TraitEngine, }; use rustc_middle::ty::{self, TyCtxt, TyVid, TypeVisitableExt, TypingMode}; +use rustc_next_trait_solver::solve::fast_path::compute_goal_fast_path; use rustc_next_trait_solver::solve::{ GoalEvaluation, GoalStalledOn, HasChanged, MaybeInfo, SolverDelegateEvalExt as _, StalledOnCoroutines, @@ -184,7 +185,22 @@ where obligation: PredicateObligation<'tcx>, ) { assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots()); - self.obligations.register(obligation, None); + + let delegate = <&SolverDelegate<'tcx>>::from(infcx); + if let Some(GoalEvaluation { goal: _, certainty, has_changed: _, stalled_on }) = + compute_goal_fast_path(delegate, obligation.as_goal(), obligation.cause.span) + { + // If we can take the fast path, don't even bother adding the goal to obligations, + // or if `Certainty::Maybe`, add it with precise stalled_on information. + match certainty { + Certainty::Yes => {} + Certainty::Maybe(_) => { + self.obligations.register(obligation, stalled_on); + } + } + } else { + self.obligations.register(obligation, None); + } } fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec { From 1cb6d00d8ae50989876281398de0979176a7db95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 9 Jul 2026 10:18:22 +0200 Subject: [PATCH 09/10] fix bug when fast paths aren't rerun when opaques change --- compiler/rustc_middle/src/traits/solve.rs | 2 +- .../src/solve/eval_ctxt/fast_path.rs | 20 +++--- .../src/solve/eval_ctxt/mod.rs | 10 ++- .../src/solve/delegate.rs | 66 +++++++++++++------ compiler/rustc_type_ir/src/solve/mod.rs | 10 +-- 5 files changed, 70 insertions(+), 38 deletions(-) diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs index c45ebb80255cc..c424ca71c0801 100644 --- a/compiler/rustc_middle/src/traits/solve.rs +++ b/compiler/rustc_middle/src/traits/solve.rs @@ -18,7 +18,7 @@ pub type FetchEligibleAssocItemResponse<'tcx> = ir::solve::FetchEligibleAssocItemResponse>; pub type ComputeGoalFastPathOutcome<'tcx> = ir::solve::ComputeGoalFastPathOutcome>; pub type GoalStalledOn<'tcx> = ir::solve::GoalStalledOn>; -pub type GoalStalledOnReason<'tcx> = ir::solve::GoalStalledOnReason>; +pub type GoalStalledOnOpaques<'tcx> = ir::solve::GoalStalledOnOpaques>; pub type SucceededInErased<'tcx> = ir::solve::SucceededInErased>; pub type PredefinedOpaques<'tcx> = &'tcx ty::List<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>; diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index 6a7d49b36b64f..a6c2bce88b031 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -7,7 +7,7 @@ use rustc_type_ir::inherent::*; use rustc_type_ir::solve::{ - Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnReason, + Certainty, ComputeGoalFastPathOutcome, Goal, GoalStalledOn, GoalStalledOnOpaques, SucceededInErased, }; use rustc_type_ir::{InferCtxtLike, Interner}; @@ -43,7 +43,7 @@ where } // If the goal isn't stalled, we should definitely run it. - let Some(&GoalStalledOn { ref reason, ref stalled_vars, ref sub_roots, stalled_certainty }) = + let Some(&GoalStalledOn { ref opaques, ref stalled_vars, ref sub_roots, stalled_certainty }) = stalled_on else { return MayMakeProgress; @@ -61,14 +61,18 @@ where return MayMakeProgress; } - match reason { - GoalStalledOnReason::FastPath => { - // fastpath is never because of opaques, we can skip this check - } - &GoalStalledOnReason::Other { num_opaques, ref previously_succeeded_in_erased } => { + match opaques { + GoalStalledOnOpaques::No => {} + &GoalStalledOnOpaques::Yes { + num_opaques_in_storage, + ref previously_succeeded_in_erased, + } => { // If any opaques changed in the opaque type storage, // rerunning might make progress so we should rerun. - if delegate.opaque_types_storage_num_entries().needs_reevaluation(num_opaques) { + if delegate + .opaque_types_storage_num_entries() + .needs_reevaluation(num_opaques_in_storage) + { // Unless this goal previously succeeded in erased mode. // If the stalled goal successfully evaluated while erasing opaque types, // and the current state of the opaque type storage is not different in a way that is diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index ded62b4c81f92..f515d0581dac8 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -39,7 +39,7 @@ use crate::solve::search_graph::SearchGraph; use crate::solve::ty::may_use_unstable_feature; use crate::solve::{ CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT, - Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnReason, HasChanged, MaybeCause, + Goal, GoalEvaluation, GoalSource, GoalStalledOn, GoalStalledOnOpaques, HasChanged, MaybeCause, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, SucceededInErased, VisibleForLeakCheck, inspect, }; @@ -753,8 +753,12 @@ where stalled_vars, sub_roots, stalled_certainty: certainty, - reason: GoalStalledOnReason::Other { - num_opaques: canonical_goal.canonical.value.predefined_opaques_in_body.len(), + opaques: GoalStalledOnOpaques::Yes { + num_opaques_in_storage: canonical_goal + .canonical + .value + .predefined_opaques_in_body + .len(), previously_succeeded_in_erased, }, } diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index 913cf28cd2426..32aaa03de646e 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -11,14 +11,14 @@ use rustc_infer::infer::canonical::{ }; use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt}; use rustc_infer::traits::solve::{ - ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, + ComputeGoalFastPathOutcome, FetchEligibleAssocItemResponse, Goal, SucceededInErased, }; use rustc_middle::traits::query::NoSolution; use rustc_middle::traits::solve::Certainty; use rustc_middle::ty::{ self, MayBeErased, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt, TypingMode, }; -use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnReason}; +use rustc_next_trait_solver::solve::{GoalStalledOn, GoalStalledOnOpaques}; use rustc_span::{DUMMY_SP, Span}; use crate::traits::{EvaluateConstErr, ObligationCause, sizedness_fast_path, specialization_graph}; @@ -50,6 +50,43 @@ impl<'tcx> SolverDelegate<'tcx> { } } +/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled +/// on a list of [`ty::GenericArg`] +fn goal_stalled_on_args<'tcx>( + stalled_vars: Vec>, +) -> ComputeGoalFastPathOutcome<'tcx> { + ComputeGoalFastPathOutcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + opaques: GoalStalledOnOpaques::No, + }, + } +} + +/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled +/// on a list of [`ty::GenericArg`] *or* the opaque type storage being nonempty. +/// +fn goal_stalled_on_args_or_nonempty_opaques<'tcx>( + stalled_vars: Vec>, +) -> ComputeGoalFastPathOutcome<'tcx> { + ComputeGoalFastPathOutcome::TriviallyStalled { + stalled_on: GoalStalledOn { + stalled_vars, + sub_roots: Vec::new(), + stalled_certainty: Certainty::AMBIGUOUS, + opaques: GoalStalledOnOpaques::Yes { + num_opaques_in_storage: 0, + // This function should only be called when not in erased mode, + // otherwise this is wrong. The `compute_goal_fast_path` does this + // through `known_no_opaque_types_in_storage` + previously_succeeded_in_erased: SucceededInErased::No, + }, + }, + } +} + impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<'tcx> { type Infcx = InferCtxt<'tcx>; type Interner = TyCtxt<'tcx>; @@ -79,19 +116,6 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< ) -> ComputeGoalFastPathOutcome<'tcx> { use ComputeGoalFastPathOutcome as Outcome; - fn stalled_with_args<'tcx>( - stalled_vars: Vec>, - ) -> ComputeGoalFastPathOutcome<'tcx> { - Outcome::TriviallyStalled { - stalled_on: GoalStalledOn { - stalled_vars, - sub_roots: Vec::new(), - stalled_certainty: Certainty::AMBIGUOUS, - reason: GoalStalledOnReason::FastPath, - }, - } - } - // FIXME(-Zassumptions-on-binders): actually handle fast path if self.tcx.assumptions_on_binders() { return Outcome::NoFastPath; @@ -110,13 +134,13 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< // FIXME: Properly consider opaques here. && self.known_no_opaque_types_in_storage() { - stalled_with_args(vec![self_ty.into()]) + goal_stalled_on_args_or_nonempty_opaques(vec![self_ty.into()]) } else if trait_pred.polarity() == ty::PredicatePolarity::Positive { match self.0.tcx.as_lang_item(trait_pred.def_id()) { Some(LangItem::Sized) | Some(LangItem::MetaSized) => { let predicate = self.resolve_vars_if_possible(goal.predicate); if sizedness_fast_path(self.tcx, predicate, goal.param_env) { - return Outcome::TriviallyHolds; + Outcome::TriviallyHolds } else { Outcome::NoFastPath } @@ -133,7 +157,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER) && self_ty.is_trivially_pure_clone_copy() { - return Outcome::TriviallyHolds; + Outcome::TriviallyHolds } else { Outcome::NoFastPath } @@ -182,7 +206,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< match (self.shallow_resolve(a).kind(), self.shallow_resolve(b).kind()) { (&ty::Infer(ty::TyVar(a_vid)), &ty::Infer(ty::TyVar(b_vid))) => { self.sub_unify_ty_vids_raw(a_vid, b_vid); - stalled_with_args(vec![a.into(), b.into()]) + goal_stalled_on_args(vec![a.into(), b.into()]) } _ => Outcome::NoFastPath, } @@ -194,7 +218,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< let arg = self.shallow_resolve_const(ct); if arg.is_ct_infer() { - stalled_with_args(vec![arg.into()]) + goal_stalled_on_args(vec![arg.into()]) } else { Outcome::NoFastPath } @@ -208,7 +232,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate< if arg.is_trivially_wf(self.tcx) { Outcome::TriviallyHolds } else if arg.is_infer() { - stalled_with_args(vec![arg.into_arg()]) + goal_stalled_on_args(vec![arg.into_arg()]) } else { Outcome::NoFastPath } diff --git a/compiler/rustc_type_ir/src/solve/mod.rs b/compiler/rustc_type_ir/src/solve/mod.rs index 5b07a23387ecb..5c8f1bab46b94 100644 --- a/compiler/rustc_type_ir/src/solve/mod.rs +++ b/compiler/rustc_type_ir/src/solve/mod.rs @@ -954,13 +954,13 @@ pub enum SucceededInErased { } #[derive_where(Clone, Debug; I: Interner)] -pub enum GoalStalledOnReason { +pub enum GoalStalledOnOpaques { /// This goal got stalled in `compute_goal_fast_path`. Usually this means /// the goal is stalled on not that much, only one or two variables, and /// definitely nothing to do with opaque types. So we don't store that information. - FastPath, - Other { - num_opaques: usize, + No, + Yes { + num_opaques_in_storage: usize, previously_succeeded_in_erased: SucceededInErased, }, } @@ -973,7 +973,7 @@ pub struct GoalStalledOn { /// The certainty that will be returned on subsequent evaluations if this /// goal remains stalled. pub stalled_certainty: Certainty, - pub reason: GoalStalledOnReason, + pub opaques: GoalStalledOnOpaques, } /// For some goals we can trivially answer some questions without going through From a572787c13610621039c2976d90272eae7348156 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jana=20D=C3=B6nszelmann?= Date: Thu, 16 Jul 2026 11:13:01 +0200 Subject: [PATCH 10/10] review comments (nits) --- .../src/solve/eval_ctxt/fast_path.rs | 7 +++++ .../src/solve/eval_ctxt/mod.rs | 27 +++++++------------ .../src/solve/delegate.rs | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs index a6c2bce88b031..f5b0b28281cf7 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/fast_path.rs @@ -108,6 +108,13 @@ where WontMakeProgress(stalled_certainty) } +/// `compute_goal_fast_path` is complicated enough that outling helps, so it gets optimized +/// separately from the caller. `compute_goal_fast_path` is the inlined version, +/// and most call sites (when adding goals) use it. However, when entering the root +/// we also want to check the fast path, and there the outlining matters. +/// +/// FIXME(perf) cold might not be worth it here, given that we shuffled some things around since it +/// mattered. #[cold] #[inline(never)] pub(super) fn compute_goal_fast_path_cold( diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs index f515d0581dac8..48a421bb5dd9b 100644 --- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs +++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs @@ -237,11 +237,10 @@ where }); } - if // No need to try the fast path if stalled_on is `None`, since we already try the fast path // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying // the fast path twice for some goals. - stalled_on.is_some() + if stalled_on.is_some() && let Some(res) = compute_goal_fast_path_cold(self, goal, span) { return Ok(res); @@ -510,11 +509,10 @@ where }); } - if // No need to try the fast path if stalled_on is `None`, since we already try the fast path // immediately when adding new goals. If we didn't check `stalled_on` here we'd be trying // the fast path twice for some goals. - stalled_on.is_some() + if stalled_on.is_some() && let Some(res) = compute_goal_fast_path_cold(self.delegate, goal, self.origin_span) { return Ok(res); @@ -1611,19 +1609,20 @@ enum RerunDecision { EagerlyPropagateToParent, } +#[tracing::instrument(ret)] fn should_rerun_after_erased_canonicalization( AccessedOpaques { reason: _, rerun }: AccessedOpaques, original_typing_mode: TypingMode, parent_opaque_types: &[(OpaqueTypeKey, I::Ty)], ) -> RerunDecision { - let parent_opaque_defids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); - let opaque_in_storage = |opaques: I::LocalDefIds, defids: SmallCopyList<_>| { - if defids.as_ref().is_empty() { + let parent_opaque_def_ids = parent_opaque_types.iter().map(|(key, _)| key.def_id.into()); + let opaque_in_storage = |opaques: I::LocalDefIds, def_ids: SmallCopyList<_>| { + if def_ids.as_ref().is_empty() { RerunDecision::No } else if opaques .iter() - .chain(parent_opaque_defids) - .any(|opaque| defids.as_ref().contains(&opaque)) + .chain(parent_opaque_def_ids) + .any(|opaque| def_ids.as_ref().contains(&opaque)) { RerunDecision::Yes } else { @@ -1638,7 +1637,7 @@ fn should_rerun_after_erased_canonicalization( } }; - let res = match (rerun, original_typing_mode) { + match (rerun, original_typing_mode) { // ============================= (RerunCondition::Never, _) => RerunDecision::No, // ============================= @@ -1692,13 +1691,7 @@ fn should_rerun_after_erased_canonicalization( TypingMode::PostBorrowck { defined_opaque_types: opaques } | TypingMode::PostTypeckUntilBorrowck { defining_opaque_types: opaques }, ) => opaque_in_storage(opaques, defids), - }; - - debug!( - "checking whether to rerun {rerun:?} in outer typing mode {original_typing_mode:?} and opaques {parent_opaque_types:?}: {res:?}" - ); - - res + } } /// Do not call this directly, use the `tcx` query instead. diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs index 32aaa03de646e..605f4d6758a5a 100644 --- a/compiler/rustc_trait_selection/src/solve/delegate.rs +++ b/compiler/rustc_trait_selection/src/solve/delegate.rs @@ -50,7 +50,7 @@ impl<'tcx> SolverDelegate<'tcx> { } } -/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled +/// Create a [`ComputeGoalFastPathOutcome`] signalling the goal is stalled /// on a list of [`ty::GenericArg`] fn goal_stalled_on_args<'tcx>( stalled_vars: Vec>,