Avoid mono collection hangs from recursive type growth

compiler/rustc_middle/src/ty/instance.rs

@@ -343,7 +343,7 @@ impl<'tcx> InstanceKind<'tcx> {
     }
 }
 
-fn type_length<'tcx>(item: impl TypeVisitable<TyCtxt<'tcx>>) -> usize {
+pub fn type_length<'tcx>(item: impl TypeVisitable<TyCtxt<'tcx>>) -> usize {
     struct Visitor<'tcx> {
         type_length: usize,
         cache: FxHashMap<Ty<'tcx>, usize>,

compiler/rustc_middle/src/ty/mod.rs

@@ -86,7 +86,7 @@ pub use self::context::{
     CtxtInterners, CurrentGcx, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed, tls,
 };
 pub use self::fold::*;
-pub use self::instance::{Instance, InstanceKind, ReifyReason};
+pub use self::instance::{Instance, InstanceKind, ReifyReason, type_length};
 pub(crate) use self::list::RawList;
 pub use self::list::{List, ListWithCachedTypeInfo};
 pub use self::opaque_types::OpaqueTypeKey;

compiler/rustc_monomorphize/src/collector.rs

@@ -227,7 +227,7 @@ use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{
     self, GenericArgs, GenericParamDefKind, Instance, InstanceKind, Ty, TyCtxt, TypeFoldable,
-    TypeVisitable, TypeVisitableExt, TypeVisitor, Unnormalized, VtblEntry,
+    TypeVisitable, TypeVisitableExt, TypeVisitor, Unnormalized, VtblEntry, type_length,
 };
 use rustc_middle::util::Providers;
 use rustc_middle::{bug, span_bug};
@@ -664,10 +664,20 @@ fn check_recursion_limit<'tcx>(
         recursion_depth
     };
 
+    // Rust code can create exponentially-long types using only a
+    // polynomial recursion depth. Start checking the type length before
+    // the depth limit is reached, to avoid hanging on enormous instance
+    // arguments.
+    let type_length_check_depth = (recursion_limit / 8).0.max(4);
+    let recursive_type_growth_limit_reached = recursion_depth >= type_length_check_depth
+        && !tcx.type_length_limit().value_within_limit(type_length(instance.args));
+
     // Code that needs to instantiate the same function recursively
-    // more than the recursion limit is assumed to be causing an
-    // infinite expansion.
-    if !recursion_limit.value_within_limit(adjusted_recursion_depth) {
+    // more than the recursion limit, or with type arguments that exceed the
+    // type length limit, is assumed to be causing an infinite expansion.
+    if !recursion_limit.value_within_limit(adjusted_recursion_depth)
+        || recursive_type_growth_limit_reached
+    {
         let def_span = tcx.def_span(def_id);
         let def_path_str = tcx.def_path_str(def_id);
         tcx.dcx().emit_fatal(RecursionLimit { span, instance, def_span, def_path_str });

tests/ui/codegen/overflow-during-mono.rs

@@ -1,7 +1,8 @@
-//~ ERROR overflow evaluating the requirement `for<'a> {closure@$DIR/overflow-during-mono.rs:14:41: 14:44}: FnMut(&'a _)`
 //@ build-fail
 //@ compile-flags: -Zwrite-long-types-to-disk=yes
 
+//~? ERROR reached the recursion limit while instantiating
+
 #![recursion_limit = "32"]
 
 fn quicksort<It: Clone + Iterator<Item = T>, I: IntoIterator<IntoIter = It>, T: Ord>(

tests/ui/codegen/overflow-during-mono.stderr

@@ -1,13 +1,10 @@
-error[E0275]: overflow evaluating the requirement `for<'a> {closure@$DIR/overflow-during-mono.rs:14:41: 14:44}: FnMut(&'a _)`
+error: reached the recursion limit while instantiating `<Filter<Filter<..., ...>, ...> as Iterator>::try_fold::<(), ..., ...>`
+  --> $SRC_DIR/core/src/iter/adapters/filter.rs:LL:COL
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "64"]` attribute to your crate (`overflow_during_mono`)
-   = note: required for `Filter<IntoIter<i32, 11>, {closure@overflow-during-mono.rs:14:41}>` to implement `Iterator`
-   = note: 31 redundant requirements hidden
-   = note: required for `Filter<Filter<Filter<Filter<Filter<..., ...>, ...>, ...>, ...>, ...>` to implement `Iterator`
-   = note: required for `Filter<Filter<Filter<Filter<Filter<..., ...>, ...>, ...>, ...>, ...>` to implement `IntoIterator`
+note: `<Filter<I, P> as Iterator>::try_fold` defined here
+  --> $SRC_DIR/core/src/iter/adapters/filter.rs:LL:COL
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/overflow-during-mono.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error
 
-For more information about this error, try `rustc --explain E0275`.

tests/ui/issues/issue-22638.rs

@@ -1,13 +1,12 @@
 //@ build-fail
 
 #![allow(unused)]
-
 #![recursion_limit = "20"]
 #![type_length_limit = "20000000"]
 #![crate_type = "rlib"]
 
 #[derive(Clone)]
-struct A (B);
+struct A(B);
 
 impl A {
     pub fn matches<F: Fn()>(&self, f: &F) {
@@ -25,34 +24,30 @@ enum B {
 impl B {
     pub fn matches<F: Fn()>(&self, f: &F) {
         match self {
-            &B::Variant2(ref factor) => {
-                factor.matches(&|| ())
-            }
-            _ => unreachable!("")
+            &B::Variant2(ref factor) => factor.matches(&|| ()),
+            _ => unreachable!(""),
         }
     }
 }
 
 #[derive(Clone)]
-struct C (D);
+struct C(D);
 
 impl C {
     pub fn matches<F: Fn()>(&self, f: &F) {
         let &C(ref base) = self;
-        base.matches(&|| {
-            C(base.clone()).matches(f)
-        })
+        base.matches(&|| C(base.clone()).matches(f))
+        //~^ ERROR reached the recursion limit while instantiating
     }
 }
 
 #[derive(Clone)]
-struct D (Box<A>);
+struct D(Box<A>);
 
 impl D {
     pub fn matches<F: Fn()>(&self, f: &F) {
         let &D(ref a) = self;
         a.matches(f)
-        //~^ ERROR reached the recursion limit while instantiating
     }
 }
 

tests/ui/issues/issue-22638.stderr

@@ -1,11 +1,11 @@
-error: reached the recursion limit while instantiating `A::matches::<{closure@$DIR/issue-22638.rs:42:23: 42:25}>`
-  --> $DIR/issue-22638.rs:54:9
+error: reached the recursion limit while instantiating `D::matches::<{closure@$DIR/issue-22638.rs:39:23: 39:25}>`
+  --> $DIR/issue-22638.rs:39:9
    |
-LL |         a.matches(f)
-   |         ^^^^^^^^^^^^
+LL |         base.matches(&|| C(base.clone()).matches(f))
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-note: `A::matches` defined here
-  --> $DIR/issue-22638.rs:13:5
+note: `D::matches` defined here
+  --> $DIR/issue-22638.rs:48:5
    |
 LL |     pub fn matches<F: Fn()>(&self, f: &F) {
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

tests/ui/recursion/issue-83150.rs

@@ -1,4 +1,3 @@
-//~ ERROR overflow evaluating the requirement `Map<&mut std::ops::Range<u8>, {closure@$DIR/issue-83150.rs:12:24: 12:27}>: Iterator`
 //@ build-fail
 //@ compile-flags: -Copt-level=0 -Zwrite-long-types-to-disk=yes
 
@@ -10,4 +9,5 @@ fn main() {
 fn func<T: Iterator<Item = u8>>(iter: &mut T) {
     //~^ WARN function cannot return without recursing
     func(&mut iter.map(|x| x + 1))
+    //~^ ERROR reached the recursion limit while instantiating
 }

tests/ui/recursion/issue-83150.stderr

@@ -1,5 +1,5 @@
 warning: function cannot return without recursing
-  --> $DIR/issue-83150.rs:10:1
+  --> $DIR/issue-83150.rs:9:1
    |
 LL | fn func<T: Iterator<Item = u8>>(iter: &mut T) {
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ cannot return without recursing
@@ -10,15 +10,19 @@ LL |     func(&mut iter.map(|x| x + 1))
    = help: a `loop` may express intention better if this is on purpose
    = note: `#[warn(unconditional_recursion)]` on by default
 
-error[E0275]: overflow evaluating the requirement `Map<&mut std::ops::Range<u8>, {closure@$DIR/issue-83150.rs:12:24: 12:27}>: Iterator`
+error: reached the recursion limit while instantiating `func::<Map<&mut Map<&mut Map<&mut Map<..., ...>, ...>, ...>, ...>>`
+  --> $DIR/issue-83150.rs:11:5
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`issue_83150`)
-   = note: required for `&mut Map<&mut Range<u8>, {closure@issue-83150.rs:12:24}>` to implement `Iterator`
-   = note: 65 redundant requirements hidden
-   = note: required for `&mut Map<&mut Map<&mut Map<&mut Map<&mut ..., ...>, ...>, ...>, ...>` to implement `Iterator`
+LL |     func(&mut iter.map(|x| x + 1))
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+note: `func` defined here
+  --> $DIR/issue-83150.rs:9:1
+   |
+LL | fn func<T: Iterator<Item = u8>>(iter: &mut T) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/issue-83150.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error; 1 warning emitted
 
-For more information about this error, try `rustc --explain E0275`.

tests/ui/recursion/recursive-mono-type-growth-issue-156272.rs

@@ -0,0 +1,22 @@
+//@ build-fail
+//@ compile-flags: -Zwrite-long-types-to-disk=yes
+
+#![allow(dead_code)]
+
+enum Foo<A> {
+    Fst,
+    Snd(Box<dyn Fn() -> Foo<(A, A)>>),
+}
+
+fn recursive<A>(x: Foo<A>) {
+    match x {
+        Foo::Fst => (),
+        Foo::Snd(f) => recursive(f()),
+        //~^ ERROR reached the recursion limit while instantiating
+    }
+}
+
+fn main() {
+    let p0: Foo<()> = Foo::Fst;
+    recursive(p0);
+}

tests/ui/recursion/recursive-mono-type-growth-issue-156272.stderr

@@ -0,0 +1,16 @@
+error: reached the recursion limit while instantiating `recursive::<(((((((..., ...), ...), ...), ...), ...), ...), ...)>`
+  --> $DIR/recursive-mono-type-growth-issue-156272.rs:14:24
+   |
+LL |         Foo::Snd(f) => recursive(f()),
+   |                        ^^^^^^^^^^^^^^
+   |
+note: `recursive` defined here
+  --> $DIR/recursive-mono-type-growth-issue-156272.rs:11:1
+   |
+LL | fn recursive<A>(x: Foo<A>) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^
+   = note: the full name for the type has been written to '$TEST_BUILD_DIR/recursive-mono-type-growth-issue-156272.long-type-$LONG_TYPE_HASH.txt'
+   = note: consider using `--verbose` to print the full type name to the console
+
+error: aborting due to 1 previous error
+

tests/ui/traits/issue-91949-hangs-on-recursion.rs

@@ -1,4 +1,3 @@
-//~ ERROR overflow evaluating the requirement `<std::iter::Empty<()> as Iterator>::Item == ()`
 //@ build-fail
 //@ compile-flags: -Zinline-mir=no -Zwrite-long-types-to-disk=yes
 
@@ -24,6 +23,7 @@ where
     T: Iterator<Item = ()>,
 {
     recurse(IteratorOfWrapped(elements).map(|t| t.0))
+    //~^ ERROR reached the recursion limit while instantiating
 }
 
 fn main() {

tests/ui/traits/issue-91949-hangs-on-recursion.stderr

@@ -1,5 +1,5 @@
 warning: function cannot return without recursing
-  --> $DIR/issue-91949-hangs-on-recursion.rs:21:1
+  --> $DIR/issue-91949-hangs-on-recursion.rs:20:1
    |
 LL | / fn recurse<T>(elements: T) -> Vec<char>
 LL | |
@@ -13,21 +13,22 @@ LL |       recurse(IteratorOfWrapped(elements).map(|t| t.0))
    = help: a `loop` may express intention better if this is on purpose
    = note: `#[warn(unconditional_recursion)]` on by default
 
-error[E0275]: overflow evaluating the requirement `<std::iter::Empty<()> as Iterator>::Item == ()`
+error: reached the recursion limit while instantiating `recurse::<Map<IteratorOfWrapped<(), Map<..., ...>>, ...>>`
+  --> $DIR/issue-91949-hangs-on-recursion.rs:25:5
    |
-   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "512"]` attribute to your crate (`issue_91949_hangs_on_recursion`)
-note: required for `IteratorOfWrapped<(), std::iter::Empty<()>>` to implement `Iterator`
-  --> $DIR/issue-91949-hangs-on-recursion.rs:14:32
+LL |     recurse(IteratorOfWrapped(elements).map(|t| t.0))
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    |
-LL | impl<T, I: Iterator<Item = T>> Iterator for IteratorOfWrapped<T, I> {
-   |                     --------   ^^^^^^^^     ^^^^^^^^^^^^^^^^^^^^^^^
-   |                     |
-   |                     unsatisfied trait bound introduced here
-   = note: 256 redundant requirements hidden
-   = note: required for `IteratorOfWrapped<(), Map<IteratorOfWrapped<(), Map<..., ...>>, ...>>` to implement `Iterator`
+note: `recurse` defined here
+  --> $DIR/issue-91949-hangs-on-recursion.rs:20:1
+   |
+LL | / fn recurse<T>(elements: T) -> Vec<char>
+LL | |
+LL | | where
+LL | |     T: Iterator<Item = ()>,
+   | |___________________________^
    = note: the full name for the type has been written to '$TEST_BUILD_DIR/issue-91949-hangs-on-recursion.long-type-$LONG_TYPE_HASH.txt'
    = note: consider using `--verbose` to print the full type name to the console
 
 error: aborting due to 1 previous error; 1 warning emitted
 
-For more information about this error, try `rustc --explain E0275`.