Prototype of scalable vectors

The representation of the element type has been changed to be a slice
rather than a zero length array. Two feature gates are now required in
core_arch unsized fn params and unsized locals.

This still leaves unsized return types being an issue. For this we are
currently bypassing some of the `Sized` trait checking to pass when
the type is scalable simd.

This still leaves the copy issue. For that we have marked scalable
simd types as trivally pure clone copy. We have still had to remove
some trait checks for the copy trait with this though as they are
still performed in certain situations.

The implementation of `transmute` is also an issue for us. For this a
new SIMD intrinsic has been created simd_reinterpret which performs a
transmute on SIMD vectors. A few intrinsics need to be able to produce
an LLVM `undef` this intrinsic will also produce that when given a
zero sized input.

compiler/rustc_abi/src/layout.rs

@@ -231,7 +231,8 @@ pub trait LayoutCalculator {
                         hide_niches(a);
                         hide_niches(b);
                     }
-                    Abi::Vector { element, count: _ } => hide_niches(element),
+                    Abi::Vector { element, .. } => hide_niches(element),
+                    Abi::ScalableVector { element, .. } => hide_niches(element),
                     Abi::Aggregate { sized: _ } => {}
                 }
                 st.largest_niche = None;

compiler/rustc_abi/src/lib.rs

@@ -41,9 +41,11 @@ bitflags! {
         // If true, the type's layout can be randomized using
         // the seed stored in `ReprOptions.layout_seed`
         const RANDOMIZE_LAYOUT   = 1 << 4;
+        const IS_SCALABLE = 1 << 5;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits
                                  | ReprFlags::IS_SIMD.bits
+                                 | ReprFlags::IS_SCALABLE.bits
                                  | ReprFlags::IS_LINEAR.bits;
     }
 }
@@ -76,6 +78,7 @@ pub struct ReprOptions {
     pub align: Option<Align>,
     pub pack: Option<Align>,
     pub flags: ReprFlags,
+    pub scalable: Option<u32>,
     /// The seed to be used for randomizing a type's layout
     ///
     /// Note: This could technically be a `u128` which would
@@ -92,6 +95,11 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_SIMD)
     }
 
+    #[inline]
+    pub fn scalable(&self) -> bool {
+        self.flags.contains(ReprFlags::IS_SCALABLE)
+    }
+
     #[inline]
     pub fn c(&self) -> bool {
         self.flags.contains(ReprFlags::IS_C)
@@ -1243,6 +1251,10 @@ pub enum Abi {
     Uninhabited,
     Scalar(Scalar),
     ScalarPair(Scalar, Scalar),
+    ScalableVector {
+        element: Scalar,
+        elt: u64,
+    },
     Vector {
         element: Scalar,
         count: u64,
@@ -1260,6 +1272,7 @@ impl Abi {
         match *self {
             Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Aggregate { sized } => !sized,
+            Abi::ScalableVector { .. } => true,
         }
     }
 
@@ -1306,7 +1319,7 @@ impl Abi {
             Abi::Vector { element, count } => {
                 cx.data_layout().vector_align(element.size(cx) * count)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1327,7 +1340,7 @@ impl Abi {
                 // to make the size a multiple of align (e.g. for vectors of size 3).
                 (element.size(cx) * count).align_to(self.inherent_align(cx)?.abi)
             }
-            Abi::Uninhabited | Abi::Aggregate { .. } => return None,
+            Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None,
         })
     }
 
@@ -1337,6 +1350,9 @@ impl Abi {
             Abi::Scalar(s) => Abi::Scalar(s.to_union()),
             Abi::ScalarPair(s1, s2) => Abi::ScalarPair(s1.to_union(), s2.to_union()),
             Abi::Vector { element, count } => Abi::Vector { element: element.to_union(), count },
+            Abi::ScalableVector { element, elt } => {
+                Abi::ScalableVector { element: element.to_union(), elt }
+            }
             Abi::Uninhabited | Abi::Aggregate { .. } => Abi::Aggregate { sized: true },
         }
     }
@@ -1620,6 +1636,11 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
         self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1
     }
 
+    /// Returns true if the size of the type is only known at runtime.
+    pub fn is_runtime_sized(&self) -> bool {
+        matches!(self.abi, Abi::ScalableVector { .. })
+    }
+
     /// Returns `true` if the type is a ZST and not unsized.
     ///
     /// Note that this does *not* imply that the type is irrelevant for layout! It can still have
@@ -1629,6 +1650,7 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutS<FieldIdx, VariantIdx> {
             Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false,
             Abi::Uninhabited => self.size.bytes() == 0,
             Abi::Aggregate { sized } => sized && self.size.bytes() == 0,
+            Abi::ScalableVector { .. } => false,
         }
     }
 

compiler/rustc_attr/messages.ftl

@@ -88,6 +88,9 @@ attr_rustc_allowed_unstable_pairing =
 attr_rustc_promotable_pairing =
     `rustc_promotable` attribute must be paired with either a `rustc_const_unstable` or a `rustc_const_stable` attribute
 
+attr_scalable_missing_n =
+    invalid `scalable(num)` attribute: `scalable` needs an argument
+    .suggestion = supply an argument here
 attr_soft_no_args =
     `soft` should not have any arguments
 

compiler/rustc_attr/src/builtin.rs

@@ -909,6 +909,7 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(u32),
+    ReprScalable(u32),
 }
 
 #[derive(Eq, PartialEq, Debug, Copy, Clone)]
@@ -964,6 +965,13 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         recognised = true;
                         None
                     }
+                    sym::scalable => {
+                        sess.emit_err(session_diagnostics::ScalableAttrMissingN {
+                            span: item.span(),
+                        });
+                        recognised = true;
+                        None
+                    }
                     name => int_type_of_word(name).map(ReprInt),
                 };
 
@@ -985,6 +993,12 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         Ok(literal) => acc.push(ReprPacked(literal)),
                         Err(message) => literal_error = Some(message),
                     };
+                } else if name == sym::scalable {
+                    recognised = true;
+                    match parse_scalable(&value.kind) {
+                        Ok(literal) => acc.push(ReprScalable(literal)),
+                        Err(message) => literal_error = Some(message),
+                    };
                 } else if matches!(name, sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
@@ -1004,7 +1018,10 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
             } else if let Some(meta_item) = item.meta_item() {
                 match &meta_item.kind {
                     MetaItemKind::NameValue(value) => {
-                        if meta_item.has_name(sym::align) || meta_item.has_name(sym::packed) {
+                        if meta_item.has_name(sym::align)
+                            || meta_item.has_name(sym::packed)
+                            || meta_item.has_name(sym::scalable)
+                        {
                             let name = meta_item.name_or_empty().to_ident_string();
                             recognised = true;
                             sess.emit_err(session_diagnostics::IncorrectReprFormatGeneric {
@@ -1199,3 +1216,11 @@ pub fn parse_confusables(attr: &Attribute) -> Option<Vec<Symbol>> {
 
     return Some(candidates);
 }
+
+pub fn parse_scalable(node: &ast::LitKind) -> Result<u32, &'static str> {
+    if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node {
+        (*literal).try_into().map_err(|_| "integer too large")
+    } else {
+        Err("not an unsuffixed integer")
+    }
+}

compiler/rustc_attr/src/session_diagnostics.rs

@@ -390,3 +390,11 @@ pub(crate) struct UnknownVersionLiteral {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(attr_scalable_missing_n, code = "E0796")]
+pub(crate) struct ScalableAttrMissingN {
+    #[primary_span]
+    #[suggestion(applicability = "has-placeholders", code = "scalable(...)")]
+    pub span: Span,
+}

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -515,7 +515,15 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             place_ty = self.sanitize_projection(place_ty, elem, place, location, context);
         }
 
-        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
+        // The Copy trait isn't implemented for scalable SIMD types.
+        // These types live somewhere between `Sized` and `Unsize`.
+        // The bounds on `Copy` disallow the trait from being
+        // implemented for them. As a result of this no bounds from
+        // `Copy` apply for the type, therefore, skipping this check
+        // should be perfectly legal.
+        if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context
+            && !place_ty.ty.is_scalable_simd()
+        {
             let tcx = self.tcx();
             let trait_ref =
                 ty::TraitRef::from_lang_item(tcx, LangItem::Copy, self.last_span, [place_ty.ty]);
@@ -1763,11 +1771,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // expressions evaluate through `as_temp` or `into` a return
             // slot or local, so to find all unsized rvalues it is enough
             // to check all temps, return slots and locals.
-            if self.reported_errors.replace((ty, span)).is_none() {
-                // While this is located in `nll::typeck` this error is not
-                // an NLL error, it's a required check to prevent creation
-                // of unsized rvalues in a call expression.
-                self.tcx().sess.emit_err(MoveUnsized { ty, span });
+            if !ty.is_scalable_simd() {
+                if self.reported_errors.replace((ty, span)).is_none() {
+                    // While this is located in `nll::typeck` this error is not
+                    // an NLL error, it's a required check to prevent creation
+                    // of unsized rvalues in a call expression.
+                    self.tcx().sess.emit_err(MoveUnsized { ty, span });
+                }
             }
         }
     }

compiler/rustc_codegen_gcc/src/abi.rs

@@ -94,6 +94,7 @@ impl GccType for Reg {
                 }
             },
             RegKind::Vector => unimplemented!(), //cx.type_vector(cx.type_i8(), self.size.bytes()),
+            RegKind::ScalableVector => unimplemented!(),
         }
     }
 }

compiler/rustc_codegen_gcc/src/intrinsic/mod.rs

@@ -274,7 +274,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     let layout = self.layout_of(tp_ty).layout;
                     let _use_integer_compare = match layout.abi() {
                         Scalar(_) | ScalarPair(_, _) => true,
-                        Uninhabited | Vector { .. } => false,
+                        Uninhabited | Vector { .. } | ScalableVector { .. } => false,
                         Aggregate { .. } => {
                             // For rusty ABIs, small aggregates are actually passed
                             // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -80,6 +80,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
                 false,
             );
         }
+        Abi::ScalableVector { .. } => todo!(),
         Abi::Uninhabited | Abi::Aggregate { .. } => {}
     }
 
@@ -158,15 +159,15 @@ pub trait LayoutGccExt<'tcx> {
 impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
     fn is_gcc_immediate(&self) -> bool {
         match self.abi {
-            Abi::Scalar(_) | Abi::Vector { .. } => true,
+            Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => true,
             Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false,
         }
     }
 
     fn is_gcc_scalar_pair(&self) -> bool {
         match self.abi {
             Abi::ScalarPair(..) => true,
-            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false,
+            Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Aggregate { .. } => false,
         }
     }
 

compiler/rustc_codegen_llvm/src/abi.rs

@@ -124,6 +124,7 @@ impl LlvmType for Reg {
                 _ => bug!("unsupported float: {:?}", self),
             },
             RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()),
+            RegKind::ScalableVector => cx.type_scalable_vector(cx.type_i8(), 16),
         }
     }
 }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -489,7 +489,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     #[instrument(level = "trace", skip(self))]
     fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
-        assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
+        assert_eq!(
+            place.llextra.is_some(),
+            place.layout.is_unsized() && !place.layout.is_runtime_sized()
+        );
 
         if place.layout.is_zst() {
             return OperandRef::zero_sized(place.layout);

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -1006,6 +1006,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
                         Cow::Borrowed(f.name.as_str())
                     };
                     let field_layout = struct_type_and_layout.field(cx, i);
+
                     build_field_di_node(
                         cx,
                         owner,

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -302,6 +302,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 let use_integer_compare = match layout.abi() {
                     Scalar(_) | ScalarPair(_, _) => true,
                     Uninhabited | Vector { .. } => false,
+                    ScalableVector { .. } => {
+                        tcx.sess.emit_err(InvalidMonomorphization::NonScalableType {
+                            span,
+                            name: sym::raw_eq,
+                            ty: tp_ty,
+                        });
+                        return;
+                    }
                     Aggregate { .. } => {
                         // For rusty ABIs, small aggregates are actually passed
                         // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@@ -983,6 +991,19 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
     }
 
+    if name == sym::simd_reinterpret {
+        require_simd!(ret_ty, SimdReturn);
+
+        use rustc_codegen_ssa::mir::operand::OperandValue;
+        return Ok(match args[0].val {
+            OperandValue::Ref(val, _, _) | OperandValue::Immediate(val) => {
+                bx.bitcast(val, llret_ty)
+            }
+            OperandValue::ZeroSized => bx.const_undef(llret_ty),
+            OperandValue::Pair(_, _) => todo!(),
+        });
+    }
+
     // every intrinsic below takes a SIMD vector as its first argument
     let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
     let in_ty = arg_tys[0];
@@ -1176,12 +1197,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
         );
         match m_elem_ty.kind() {
-            ty::Int(_) => {}
+            ty::Int(_) | ty::Bool => {}
             _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
         }
         // truncate the mask to a vector of i1s
         let i1 = bx.type_i1();
-        let i1xn = bx.type_vector(i1, m_len as u64);
+        let i1xn = if arg_tys[1].is_scalable_simd() {
+            bx.type_scalable_vector(i1, m_len as u64)
+        } else {
+            bx.type_vector(i1, m_len as u64)
+        };
         let m_i1s = bx.trunc(args[0].immediate(), i1xn);
         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
     }
@@ -1952,6 +1977,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             out_elem
         });
     }
+
     macro_rules! arith_binary {
         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
             $(if name == sym::$name {

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -882,6 +882,7 @@ extern "C" {
     pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type;
     pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
     pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
+    pub fn LLVMScalableVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
 
     pub fn LLVMGetElementType(Ty: &Type) -> &Type;
     pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint;

compiler/rustc_codegen_llvm/src/type_.rs

@@ -69,6 +69,10 @@ impl<'ll> CodegenCx<'ll, '_> {
         unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
     }
 
+    pub(crate) fn type_scalable_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type {
+        unsafe { llvm::LLVMScalableVectorType(ty, len as c_uint) }
+    }
+
     pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> {
         unsafe {
             let n_args = llvm::LLVMCountParamTypes(ty) as usize;