Auto merge of #143924 - davidtwco:sve-infrastructure, r=workingjubilee

`rustc_scalable_vector(N)`

Supercedes #118917.

Initial experimental implementation of rust-lang/rfcs#3838. Introduces a `rustc_scalable_vector(N)` attribute that can be applied to types with a single `[$ty]` field (for `u{16,32,64}`, `i{16,32,64}`, `f{32,64}`, `bool`). `rustc_scalable_vector` types are lowered to scalable vectors in the codegen backend.

As with any unstable feature, there will necessarily be follow-ups as we experiment and find cases that we've not considered or still need some logic to handle, but this aims to be a decent baseline to start from.

See #145052 for request for a lang experiment.

Author: bors

compiler/rustc_abi/src/callconv.rs

@@ -60,6 +60,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
     /// This is public so that it can be used in unit tests, but
     /// should generally only be relevant to the ABI details of
     /// specific targets.
+    #[tracing::instrument(skip(cx), level = "debug")]
     pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, Heterogeneous>
     where
         Ty: TyAbiInterface<'a, C> + Copy,
@@ -82,6 +83,10 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
                 }))
             }
 
+            BackendRepr::ScalableVector { .. } => {
+                unreachable!("`homogeneous_aggregate` should not be called for scalable vectors")
+            }
+
             BackendRepr::ScalarPair(..) | BackendRepr::Memory { sized: true } => {
                 // Helper for computing `homogeneous_aggregate`, allowing a custom
                 // starting offset (used below for handling variants).

compiler/rustc_abi/src/layout.rs

@@ -11,7 +11,7 @@ use tracing::{debug, trace};
 use crate::{
     AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
     LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
-    Variants, WrappingRange,
+    TargetDataLayout, Variants, WrappingRange,
 };
 
 mod coroutine;
@@ -143,58 +143,32 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         })
     }
 
-    pub fn simd_type<
+    pub fn scalable_vector_type<FieldIdx, VariantIdx, F>(
+        &self,
+        element: F,
+        count: u64,
+    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+    where
         FieldIdx: Idx,
         VariantIdx: Idx,
         F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
-    >(
+    {
+        vector_type_layout(VectorKind::Scalable, self.cx.data_layout(), element, count)
+    }
+
+    pub fn simd_type<FieldIdx, VariantIdx, F>(
         &self,
         element: F,
         count: u64,
         repr_packed: bool,
-    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
-        let elt = element.as_ref();
-        if count == 0 {
-            return Err(LayoutCalculatorError::ZeroLengthSimdType);
-        } else if count > crate::MAX_SIMD_LANES {
-            return Err(LayoutCalculatorError::OversizedSimdType {
-                max_lanes: crate::MAX_SIMD_LANES,
-            });
-        }
-
-        let BackendRepr::Scalar(e_repr) = elt.backend_repr else {
-            return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
-        };
-
-        // Compute the size and alignment of the vector
-        let dl = self.cx.data_layout();
-        let size =
-            elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
-        let (repr, align) = if repr_packed && !count.is_power_of_two() {
-            // Non-power-of-two vectors have padding up to the next power-of-two.
-            // If we're a packed repr, remove the padding while keeping the alignment as close
-            // to a vector as possible.
-            (BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
-        } else {
-            (BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
-        };
-        let size = size.align_to(align);
-
-        Ok(LayoutData {
-            variants: Variants::Single { index: VariantIdx::new(0) },
-            fields: FieldsShape::Arbitrary {
-                offsets: [Size::ZERO].into(),
-                memory_index: [0].into(),
-            },
-            backend_repr: repr,
-            largest_niche: elt.largest_niche,
-            uninhabited: false,
-            size,
-            align: AbiAlign::new(align),
-            max_repr_align: None,
-            unadjusted_abi_align: elt.align.abi,
-            randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
-        })
+    ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+    where
+        FieldIdx: Idx,
+        VariantIdx: Idx,
+        F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
+    {
+        let kind = if repr_packed { VectorKind::PackedFixed } else { VectorKind::Fixed };
+        vector_type_layout(kind, self.cx.data_layout(), element, count)
     }
 
     /// Compute the layout for a coroutine.
@@ -453,6 +427,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                 BackendRepr::Scalar(..)
                 | BackendRepr::ScalarPair(..)
                 | BackendRepr::SimdVector { .. }
+                | BackendRepr::ScalableVector { .. }
                 | BackendRepr::Memory { .. } => repr,
             },
         };
@@ -524,7 +499,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                     hide_niches(a);
                     hide_niches(b);
                 }
-                BackendRepr::SimdVector { element, count: _ } => hide_niches(element),
+                BackendRepr::SimdVector { element, .. }
+                | BackendRepr::ScalableVector { element, .. } => hide_niches(element),
                 BackendRepr::Memory { sized: _ } => {}
             }
             st.largest_niche = None;
@@ -1501,3 +1477,67 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         s
     }
 }
+
+enum VectorKind {
+    /// `#[rustc_scalable_vector]`
+    Scalable,
+    /// `#[repr(simd, packed)]`
+    PackedFixed,
+    /// `#[repr(simd)]`
+    Fixed,
+}
+
+fn vector_type_layout<FieldIdx, VariantIdx, F>(
+    kind: VectorKind,
+    dl: &TargetDataLayout,
+    element: F,
+    count: u64,
+) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F>
+where
+    FieldIdx: Idx,
+    VariantIdx: Idx,
+    F: AsRef<LayoutData<FieldIdx, VariantIdx>> + fmt::Debug,
+{
+    let elt = element.as_ref();
+    if count == 0 {
+        return Err(LayoutCalculatorError::ZeroLengthSimdType);
+    } else if count > crate::MAX_SIMD_LANES {
+        return Err(LayoutCalculatorError::OversizedSimdType { max_lanes: crate::MAX_SIMD_LANES });
+    }
+
+    let BackendRepr::Scalar(element) = elt.backend_repr else {
+        return Err(LayoutCalculatorError::NonPrimitiveSimdType(element));
+    };
+
+    // Compute the size and alignment of the vector
+    let size =
+        elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?;
+    let (repr, align) = match kind {
+        VectorKind::Scalable => {
+            (BackendRepr::ScalableVector { element, count }, dl.llvmlike_vector_align(size))
+        }
+        // Non-power-of-two vectors have padding up to the next power-of-two.
+        // If we're a packed repr, remove the padding while keeping the alignment as close
+        // to a vector as possible.
+        VectorKind::PackedFixed if !count.is_power_of_two() => {
+            (BackendRepr::Memory { sized: true }, Align::max_aligned_factor(size))
+        }
+        VectorKind::PackedFixed | VectorKind::Fixed => {
+            (BackendRepr::SimdVector { element, count }, dl.llvmlike_vector_align(size))
+        }
+    };
+    let size = size.align_to(align);
+
+    Ok(LayoutData {
+        variants: Variants::Single { index: VariantIdx::new(0) },
+        fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() },
+        backend_repr: repr,
+        largest_niche: elt.largest_niche,
+        uninhabited: false,
+        size,
+        align: AbiAlign::new(align),
+        max_repr_align: None,
+        unadjusted_abi_align: elt.align.abi,
+        randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)),
+    })
+}

compiler/rustc_abi/src/layout/ty.rs

@@ -155,7 +155,7 @@ impl<'a, Ty> AsRef<LayoutData<FieldIdx, VariantIdx>> for TyAndLayout<'a, Ty> {
 
 /// Trait that needs to be implemented by the higher-level type representation
 /// (e.g. `rustc_middle::ty::Ty`), to provide `rustc_target::abi` functionality.
-pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug {
+pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug + std::fmt::Display {
     fn ty_and_layout_for_variant(
         this: TyAndLayout<'a, Self>,
         cx: &C,
@@ -172,6 +172,7 @@ pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug {
     fn is_tuple(this: TyAndLayout<'a, Self>) -> bool;
     fn is_unit(this: TyAndLayout<'a, Self>) -> bool;
     fn is_transparent(this: TyAndLayout<'a, Self>) -> bool;
+    fn is_scalable_vector(this: TyAndLayout<'a, Self>) -> bool;
     /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
     fn is_pass_indirectly_in_non_rustic_abis_flag_set(this: TyAndLayout<'a, Self>) -> bool;
 }
@@ -271,6 +272,13 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         Ty::is_transparent(self)
     }
 
+    pub fn is_scalable_vector<C>(self) -> bool
+    where
+        Ty: TyAbiInterface<'a, C>,
+    {
+        Ty::is_scalable_vector(self)
+    }
+
     /// If this method returns `true`, then this type should always have a `PassMode` of
     /// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a
     /// non-Rustic ABI (this is true for structs annotated with the

compiler/rustc_abi/src/lib.rs

@@ -96,9 +96,11 @@ bitflags! {
         /// If true, the type is always passed indirectly by non-Rustic ABIs.
         /// See [`TyAndLayout::pass_indirectly_in_non_rustic_abis`] for details.
         const PASS_INDIRECTLY_IN_NON_RUSTIC_ABIS = 1 << 5;
-        /// Any of these flags being set prevent field reordering optimisation.
-        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
+        const IS_SCALABLE        = 1 << 6;
+         // Any of these flags being set prevent field reordering optimisation.
+        const FIELD_ORDER_UNOPTIMIZABLE = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
+                                 | ReprFlags::IS_SCALABLE.bits()
                                  | ReprFlags::IS_LINEAR.bits();
         const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
@@ -135,6 +137,19 @@ impl IntegerType {
     }
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+#[cfg_attr(
+    feature = "nightly",
+    derive(Encodable_NoContext, Decodable_NoContext, HashStable_Generic)
+)]
+pub enum ScalableElt {
+    /// `N` in `rustc_scalable_vector(N)` - the element count of the scalable vector
+    ElementCount(u16),
+    /// `rustc_scalable_vector` w/out `N`, used for tuple types of scalable vectors that only
+    /// contain other scalable vectors
+    Container,
+}
+
 /// Represents the repr options provided by the user.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Default)]
 #[cfg_attr(
@@ -146,6 +161,8 @@ pub struct ReprOptions {
     pub align: Option<Align>,
     pub pack: Option<Align>,
     pub flags: ReprFlags,
+    /// `#[rustc_scalable_vector]`
+    pub scalable: Option<ScalableElt>,
     /// The seed to be used for randomizing a type's layout
     ///
     /// Note: This could technically be a `u128` which would
@@ -162,6 +179,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_SIMD)
     }
 
+    #[inline]
+    pub fn scalable(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_SCALABLE)
+    }
+
     #[inline]
     pub fn c(&self) -> bool {
         self.flags.contains(ReprFlags::IS_C)
@@ -1736,6 +1758,10 @@ impl AddressSpace {
 pub enum BackendRepr {
     Scalar(Scalar),
     ScalarPair(Scalar, Scalar),
+    ScalableVector {
+        element: Scalar,
+        count: u64,
+    },
     SimdVector {
         element: Scalar,
         count: u64,
@@ -1754,6 +1780,12 @@ impl BackendRepr {
         match *self {
             BackendRepr::Scalar(_)
             | BackendRepr::ScalarPair(..)
+            // FIXME(rustc_scalable_vector): Scalable vectors are `Sized` while the
+            // `sized_hierarchy` feature is not yet fully implemented. After `sized_hierarchy` is
+            // fully implemented, scalable vectors will remain `Sized`, they just won't be
+            // `const Sized` - whether `is_unsized` continues to return `false` at that point will
+            // need to be revisited and will depend on what `is_unsized` is used for.
+            | BackendRepr::ScalableVector { .. }
             | BackendRepr::SimdVector { .. } => false,
             BackendRepr::Memory { sized } => !sized,
         }
@@ -1794,7 +1826,9 @@ impl BackendRepr {
             BackendRepr::Scalar(s) => Some(s.align(cx).abi),
             BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
             // The align of a Vector can vary in surprising ways
-            BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
+            BackendRepr::SimdVector { .. }
+            | BackendRepr::Memory { .. }
+            | BackendRepr::ScalableVector { .. } => None,
         }
     }
 
@@ -1816,7 +1850,9 @@ impl BackendRepr {
                 Some(size)
             }
             // The size of a Vector can vary in surprising ways
-            BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => None,
+            BackendRepr::SimdVector { .. }
+            | BackendRepr::Memory { .. }
+            | BackendRepr::ScalableVector { .. } => None,
         }
     }
 
@@ -1831,6 +1867,9 @@ impl BackendRepr {
                 BackendRepr::SimdVector { element: element.to_union(), count }
             }
             BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
+            BackendRepr::ScalableVector { element, count } => {
+                BackendRepr::ScalableVector { element: element.to_union(), count }
+            }
         }
     }
 
@@ -2071,7 +2110,9 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
     /// Returns `true` if this is an aggregate type (including a ScalarPair!)
     pub fn is_aggregate(&self) -> bool {
         match self.backend_repr {
-            BackendRepr::Scalar(_) | BackendRepr::SimdVector { .. } => false,
+            BackendRepr::Scalar(_)
+            | BackendRepr::SimdVector { .. }
+            | BackendRepr::ScalableVector { .. } => false,
             BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
         }
     }
@@ -2165,6 +2206,19 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
         self.is_sized() && self.size.bytes() == 0 && self.align.bytes() == 1
     }
 
+    /// Returns `true` if the size of the type is only known at runtime.
+    pub fn is_runtime_sized(&self) -> bool {
+        matches!(self.backend_repr, BackendRepr::ScalableVector { .. })
+    }
+
+    /// Returns the elements count of a scalable vector.
+    pub fn scalable_vector_element_count(&self) -> Option<u64> {
+        match self.backend_repr {
+            BackendRepr::ScalableVector { count, .. } => Some(count),
+            _ => None,
+        }
+    }
+
     /// Returns `true` if the type is a ZST and not unsized.
     ///
     /// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -2173,6 +2227,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
         match self.backend_repr {
             BackendRepr::Scalar(_)
             | BackendRepr::ScalarPair(..)
+            | BackendRepr::ScalableVector { .. }
             | BackendRepr::SimdVector { .. } => false,
             BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
         }

compiler/rustc_ast_passes/messages.ftl

@@ -270,6 +270,8 @@ ast_passes_precise_capturing_duplicated = duplicate `use<...>` precise capturing
 
 ast_passes_precise_capturing_not_allowed_here = `use<...>` precise capturing syntax not allowed in {$loc}
 
+ast_passes_scalable_vector_not_tuple_struct = scalable vectors must be tuple structs
+
 ast_passes_static_without_body =
     free static item without body
     .suggestion = provide a definition for the static

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -1319,6 +1319,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Struct(ident, generics, vdata) => {
                 self.with_tilde_const(Some(TildeConstReason::Struct { span: item.span }), |this| {
+                    // Scalable vectors can only be tuple structs
+                    let is_scalable_vector =
+                        item.attrs.iter().any(|attr| attr.has_name(sym::rustc_scalable_vector));
+                    if is_scalable_vector && !matches!(vdata, VariantData::Tuple(..)) {
+                        this.dcx()
+                            .emit_err(errors::ScalableVectorNotTupleStruct { span: item.span });
+                    }
+
                     match vdata {
                         VariantData::Struct { fields, .. } => {
                             this.visit_attrs_vis_ident(&item.attrs, &item.vis, ident);