Fix (A)Rc<[T]> trying to allocate RcBox larger than isize::MAX bytes

Due to ptr::add restrictions allocations must be <= isize::MAX bytes. The default implementation goes
through Vec which ensures this, but the uninit and TrustedLen code paths lack the necessary check.

Author: the8472

library/alloc/src/raw_vec.rs

@@ -513,6 +513,6 @@ fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> {
 // ensure that the code generation related to these panics is minimal as there's
 // only one location which panics rather than a bunch throughout the module.
 #[cfg(not(no_global_oom_handling))]
-fn capacity_overflow() -> ! {
+pub(crate) fn capacity_overflow() -> ! {
     panic!("capacity overflow");
 }

library/alloc/src/rc.rs

@@ -1291,6 +1291,9 @@ impl<T: ?Sized> Rc<T> {
         // `&*(ptr as *const RcBox<T>)`, but this created a misaligned
         // reference (see #54908).
         let layout = Layout::new::<RcBox<()>>().extend(value_layout).unwrap().0.pad_to_align();
+        if layout.size() > isize::MAX as usize {
+            crate::raw_vec::capacity_overflow();
+        }
         unsafe {
             Rc::try_allocate_for_layout(value_layout, allocate, mem_to_rcbox)
                 .unwrap_or_else(|_| handle_alloc_error(layout))
@@ -1314,7 +1317,9 @@ impl<T: ?Sized> Rc<T> {
         // `&*(ptr as *const RcBox<T>)`, but this created a misaligned
         // reference (see #54908).
         let layout = Layout::new::<RcBox<()>>().extend(value_layout).unwrap().0.pad_to_align();
-
+        if layout.size() > isize::MAX as usize {
+            return Err(AllocError);
+        }
         // Allocate for the layout.
         let ptr = allocate(layout)?;
 
@@ -2050,7 +2055,7 @@ impl<T, I: iter::TrustedLen<Item = T>> ToRcSlice<T> for I {
             // length exceeding `usize::MAX`.
             // The default implementation would collect into a vec which would panic.
             // Thus we panic here immediately without invoking `Vec` code.
-            panic!("capacity overflow");
+            crate::raw_vec::capacity_overflow();
         }
     }
 }

library/alloc/src/rc/tests.rs

@@ -1,4 +1,5 @@
 use super::*;
+use core::mem::MaybeUninit;
 
 use std::boxed::Box;
 use std::cell::RefCell;
@@ -462,6 +463,21 @@ fn test_from_vec() {
     assert_eq!(&r[..], [1, 2, 3]);
 }
 
+#[test]
+#[should_panic]
+fn test_uninit_slice_max_allocation() {
+    let _: Rc<[MaybeUninit<u8>]> = Rc::new_uninit_slice(isize::MAX as usize);
+}
+
+#[test]
+#[should_panic]
+fn test_from_iter_max_allocation() {
+    // isize::MAX is the max allocation size
+    // but due to the internal RcBox overhead the actual allocation will be larger and thus panic
+    let len = isize::MAX as usize;
+    let _ = (0..len).map(|_| MaybeUninit::<u8>::uninit()).collect::<Rc<[_]>>();
+}
+
 #[test]
 fn test_downcast() {
     use std::any::Any;

library/alloc/src/sync.rs

@@ -1137,6 +1137,9 @@ impl<T: ?Sized> Arc<T> {
         // `&*(ptr as *const ArcInner<T>)`, but this created a misaligned
         // reference (see #54908).
         let layout = Layout::new::<ArcInner<()>>().extend(value_layout).unwrap().0.pad_to_align();
+        if layout.size() > isize::MAX as usize {
+            crate::raw_vec::capacity_overflow();
+        }
         unsafe {
             Arc::try_allocate_for_layout(value_layout, allocate, mem_to_arcinner)
                 .unwrap_or_else(|_| handle_alloc_error(layout))
@@ -1159,7 +1162,9 @@ impl<T: ?Sized> Arc<T> {
         // `&*(ptr as *const ArcInner<T>)`, but this created a misaligned
         // reference (see #54908).
         let layout = Layout::new::<ArcInner<()>>().extend(value_layout).unwrap().0.pad_to_align();
-
+        if layout.size() > isize::MAX as usize {
+            return Err(AllocError);
+        }
         let ptr = allocate(layout)?;
 
         // Initialize the ArcInner
@@ -2648,7 +2653,7 @@ impl<T, I: iter::TrustedLen<Item = T>> ToArcSlice<T> for I {
             // length exceeding `usize::MAX`.
             // The default implementation would collect into a vec which would panic.
             // Thus we panic here immediately without invoking `Vec` code.
-            panic!("capacity overflow");
+            crate::raw_vec::capacity_overflow();
         }
     }
 }

library/alloc/src/sync/tests.rs

@@ -4,6 +4,7 @@ use std::boxed::Box;
 use std::clone::Clone;
 use std::convert::{From, TryInto};
 use std::mem::drop;
+use std::mem::MaybeUninit;
 use std::ops::Drop;
 use std::option::Option::{self, None, Some};
 use std::sync::atomic::{
@@ -520,6 +521,21 @@ fn test_from_vec() {
     assert_eq!(&r[..], [1, 2, 3]);
 }
 
+#[test]
+#[should_panic]
+fn test_uninit_slice_max_allocation() {
+    let _: Arc<[MaybeUninit<u8>]> = Arc::new_uninit_slice(isize::MAX as usize);
+}
+
+#[test]
+#[should_panic]
+fn test_from_iter_max_allocation() {
+    // isize::MAX is the max allocation size
+    // but due to the internal RcBox overhead the actual allocation will be larger and thus panic
+    let len = isize::MAX;
+    let _ = (0..len).map(|_| MaybeUninit::<u8>::uninit()).collect::<Arc<[_]>>();
+}
+
 #[test]
 fn test_downcast() {
     use std::any::Any;