From 4c14a6d4dc946d797046de4fd6d4f942cd22825e Mon Sep 17 00:00:00 2001 From: Jamie Cunliffe Date: Tue, 15 Aug 2023 15:17:29 +0100 Subject: [PATCH] Prototype of scalable vectors The representation of the element type has been changed to be a slice rather than a zero length array. Two feature gates are now required in core_arch unsized fn params and unsized locals. This still leaves unsized return types being an issue. For this we are currently bypassing some of the `Sized` trait checking to pass when the type is scalable simd. This still leaves the copy issue. For that we have marked scalable simd types as trivally pure clone copy. We have still had to remove some trait checks for the copy trait with this though as they are still performed in certain situations. The implementation of `transmute` is also an issue for us. For this a new SIMD intrinsic has been created simd_reinterpret which performs a transmute on SIMD vectors. A few intrinsics need to be able to produce an LLVM `undef` this intrinsic will also produce that when given a zero sized input. --- compiler/rustc_abi/src/layout.rs | 3 +- compiler/rustc_abi/src/lib.rs | 26 ++++++++++++++-- compiler/rustc_attr/messages.ftl | 3 ++ compiler/rustc_attr/src/builtin.rs | 27 ++++++++++++++++- .../rustc_attr/src/session_diagnostics.rs | 8 +++++ compiler/rustc_borrowck/src/type_check/mod.rs | 22 ++++++++++---- compiler/rustc_codegen_gcc/src/abi.rs | 1 + .../rustc_codegen_gcc/src/intrinsic/mod.rs | 2 +- compiler/rustc_codegen_gcc/src/type_of.rs | 5 ++-- compiler/rustc_codegen_llvm/src/abi.rs | 1 + compiler/rustc_codegen_llvm/src/builder.rs | 5 +++- .../src/debuginfo/metadata.rs | 1 + compiler/rustc_codegen_llvm/src/intrinsic.rs | 30 +++++++++++++++++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 1 + compiler/rustc_codegen_llvm/src/type_.rs | 4 +++ compiler/rustc_codegen_llvm/src/type_of.rs | 17 +++++++++-- compiler/rustc_codegen_ssa/messages.ftl | 1 + compiler/rustc_codegen_ssa/src/errors.rs | 8 +++++ .../rustc_codegen_ssa/src/mir/debuginfo.rs | 5 ++++ compiler/rustc_codegen_ssa/src/mir/operand.rs | 5 +++- compiler/rustc_codegen_ssa/src/mir/place.rs | 9 ++++-- compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 1 + .../src/interpret/validity.rs | 2 +- .../src/util/check_validity_requirement.rs | 1 + compiler/rustc_error_codes/src/error_codes.rs | 1 + .../src/error_codes/E0796.md | 10 +++++++ .../rustc_hir_analysis/src/check/check.rs | 9 +++--- .../rustc_hir_analysis/src/check/intrinsic.rs | 1 + compiler/rustc_hir_typeck/src/check.rs | 9 +++++- compiler/rustc_hir_typeck/src/expr.rs | 7 ++++- compiler/rustc_hir_typeck/src/lib.rs | 5 +++- compiler/rustc_middle/src/ty/mod.rs | 14 ++++++++- compiler/rustc_middle/src/ty/sty.rs | 18 +++++++++++ .../src/build/expr/as_operand.rs | 6 ++-- compiler/rustc_passes/src/check_attr.rs | 3 ++ compiler/rustc_span/src/symbol.rs | 3 ++ compiler/rustc_target/src/abi/call/aarch64.rs | 1 + compiler/rustc_target/src/abi/call/arm.rs | 1 + .../rustc_target/src/abi/call/loongarch.rs | 4 ++- compiler/rustc_target/src/abi/call/mod.rs | 12 +++++++- .../rustc_target/src/abi/call/powerpc64.rs | 1 + compiler/rustc_target/src/abi/call/riscv.rs | 4 ++- compiler/rustc_target/src/abi/call/x86.rs | 3 ++ compiler/rustc_target/src/abi/call/x86_64.rs | 2 +- .../rustc_target/src/abi/call/x86_win64.rs | 1 + compiler/rustc_ty_utils/src/layout.rs | 21 ++++++++++++- .../rustc_ty_utils/src/layout_sanity_check.rs | 4 +-- tests/ui/thir-print/thir-tree-match.stdout | 8 ++--- 48 files changed, 292 insertions(+), 44 deletions(-) create mode 100644 compiler/rustc_error_codes/src/error_codes/E0796.md diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 815edcc0dc4b9..0608c1b49e44c 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -231,7 +231,8 @@ pub trait LayoutCalculator { hide_niches(a); hide_niches(b); } - Abi::Vector { element, count: _ } => hide_niches(element), + Abi::Vector { element, .. } => hide_niches(element), + Abi::ScalableVector { element, .. } => hide_niches(element), Abi::Aggregate { sized: _ } => {} } st.largest_niche = None; diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index eb42803f93e4e..4039a327ad9de 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -41,9 +41,11 @@ bitflags! { // If true, the type's layout can be randomized using // the seed stored in `ReprOptions.layout_seed` const RANDOMIZE_LAYOUT = 1 << 4; + const IS_SCALABLE = 1 << 5; // Any of these flags being set prevent field reordering optimisation. const IS_UNOPTIMISABLE = ReprFlags::IS_C.bits | ReprFlags::IS_SIMD.bits + | ReprFlags::IS_SCALABLE.bits | ReprFlags::IS_LINEAR.bits; } } @@ -76,6 +78,7 @@ pub struct ReprOptions { pub align: Option, pub pack: Option, pub flags: ReprFlags, + pub scalable: Option, /// The seed to be used for randomizing a type's layout /// /// Note: This could technically be a `u128` which would @@ -92,6 +95,11 @@ impl ReprOptions { self.flags.contains(ReprFlags::IS_SIMD) } + #[inline] + pub fn scalable(&self) -> bool { + self.flags.contains(ReprFlags::IS_SCALABLE) + } + #[inline] pub fn c(&self) -> bool { self.flags.contains(ReprFlags::IS_C) @@ -1243,6 +1251,10 @@ pub enum Abi { Uninhabited, Scalar(Scalar), ScalarPair(Scalar, Scalar), + ScalableVector { + element: Scalar, + elt: u64, + }, Vector { element: Scalar, count: u64, @@ -1260,6 +1272,7 @@ impl Abi { match *self { Abi::Uninhabited | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false, Abi::Aggregate { sized } => !sized, + Abi::ScalableVector { .. } => true, } } @@ -1306,7 +1319,7 @@ impl Abi { Abi::Vector { element, count } => { cx.data_layout().vector_align(element.size(cx) * count) } - Abi::Uninhabited | Abi::Aggregate { .. } => return None, + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None, }) } @@ -1327,7 +1340,7 @@ impl Abi { // to make the size a multiple of align (e.g. for vectors of size 3). (element.size(cx) * count).align_to(self.inherent_align(cx)?.abi) } - Abi::Uninhabited | Abi::Aggregate { .. } => return None, + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => return None, }) } @@ -1337,6 +1350,9 @@ impl Abi { Abi::Scalar(s) => Abi::Scalar(s.to_union()), Abi::ScalarPair(s1, s2) => Abi::ScalarPair(s1.to_union(), s2.to_union()), Abi::Vector { element, count } => Abi::Vector { element: element.to_union(), count }, + Abi::ScalableVector { element, elt } => { + Abi::ScalableVector { element: element.to_union(), elt } + } Abi::Uninhabited | Abi::Aggregate { .. } => Abi::Aggregate { sized: true }, } } @@ -1620,6 +1636,11 @@ impl LayoutS { self.is_sized() && self.size.bytes() == 0 && self.align.abi.bytes() == 1 } + /// Returns true if the size of the type is only known at runtime. + pub fn is_runtime_sized(&self) -> bool { + matches!(self.abi, Abi::ScalableVector { .. }) + } + /// Returns `true` if the type is a ZST and not unsized. /// /// Note that this does *not* imply that the type is irrelevant for layout! It can still have @@ -1629,6 +1650,7 @@ impl LayoutS { Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } => false, Abi::Uninhabited => self.size.bytes() == 0, Abi::Aggregate { sized } => sized && self.size.bytes() == 0, + Abi::ScalableVector { .. } => false, } } diff --git a/compiler/rustc_attr/messages.ftl b/compiler/rustc_attr/messages.ftl index 7281282fec37d..9822e55e070ac 100644 --- a/compiler/rustc_attr/messages.ftl +++ b/compiler/rustc_attr/messages.ftl @@ -88,6 +88,9 @@ attr_rustc_allowed_unstable_pairing = attr_rustc_promotable_pairing = `rustc_promotable` attribute must be paired with either a `rustc_const_unstable` or a `rustc_const_stable` attribute +attr_scalable_missing_n = + invalid `scalable(num)` attribute: `scalable` needs an argument + .suggestion = supply an argument here attr_soft_no_args = `soft` should not have any arguments diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs index 7e87d1c31301b..8a7193836b310 100644 --- a/compiler/rustc_attr/src/builtin.rs +++ b/compiler/rustc_attr/src/builtin.rs @@ -909,6 +909,7 @@ pub enum ReprAttr { ReprSimd, ReprTransparent, ReprAlign(u32), + ReprScalable(u32), } #[derive(Eq, PartialEq, Debug, Copy, Clone)] @@ -964,6 +965,13 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec { recognised = true; None } + sym::scalable => { + sess.emit_err(session_diagnostics::ScalableAttrMissingN { + span: item.span(), + }); + recognised = true; + None + } name => int_type_of_word(name).map(ReprInt), }; @@ -985,6 +993,12 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec { Ok(literal) => acc.push(ReprPacked(literal)), Err(message) => literal_error = Some(message), }; + } else if name == sym::scalable { + recognised = true; + match parse_scalable(&value.kind) { + Ok(literal) => acc.push(ReprScalable(literal)), + Err(message) => literal_error = Some(message), + }; } else if matches!(name, sym::C | sym::simd | sym::transparent) || int_type_of_word(name).is_some() { @@ -1004,7 +1018,10 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec { } else if let Some(meta_item) = item.meta_item() { match &meta_item.kind { MetaItemKind::NameValue(value) => { - if meta_item.has_name(sym::align) || meta_item.has_name(sym::packed) { + if meta_item.has_name(sym::align) + || meta_item.has_name(sym::packed) + || meta_item.has_name(sym::scalable) + { let name = meta_item.name_or_empty().to_ident_string(); recognised = true; sess.emit_err(session_diagnostics::IncorrectReprFormatGeneric { @@ -1199,3 +1216,11 @@ pub fn parse_confusables(attr: &Attribute) -> Option> { return Some(candidates); } + +pub fn parse_scalable(node: &ast::LitKind) -> Result { + if let ast::LitKind::Int(literal, ast::LitIntType::Unsuffixed) = node { + (*literal).try_into().map_err(|_| "integer too large") + } else { + Err("not an unsuffixed integer") + } +} diff --git a/compiler/rustc_attr/src/session_diagnostics.rs b/compiler/rustc_attr/src/session_diagnostics.rs index ca9bbd28b9556..95a729bb2b387 100644 --- a/compiler/rustc_attr/src/session_diagnostics.rs +++ b/compiler/rustc_attr/src/session_diagnostics.rs @@ -390,3 +390,11 @@ pub(crate) struct UnknownVersionLiteral { #[primary_span] pub span: Span, } + +#[derive(Diagnostic)] +#[diag(attr_scalable_missing_n, code = "E0796")] +pub(crate) struct ScalableAttrMissingN { + #[primary_span] + #[suggestion(applicability = "has-placeholders", code = "scalable(...)")] + pub span: Span, +} diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index fdc710c4b4f32..5a021abff1beb 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -515,7 +515,15 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { place_ty = self.sanitize_projection(place_ty, elem, place, location, context); } - if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context { + // The Copy trait isn't implemented for scalable SIMD types. + // These types live somewhere between `Sized` and `Unsize`. + // The bounds on `Copy` disallow the trait from being + // implemented for them. As a result of this no bounds from + // `Copy` apply for the type, therefore, skipping this check + // should be perfectly legal. + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context + && !place_ty.ty.is_scalable_simd() + { let tcx = self.tcx(); let trait_ref = ty::TraitRef::from_lang_item(tcx, LangItem::Copy, self.last_span, [place_ty.ty]); @@ -1763,11 +1771,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // expressions evaluate through `as_temp` or `into` a return // slot or local, so to find all unsized rvalues it is enough // to check all temps, return slots and locals. - if self.reported_errors.replace((ty, span)).is_none() { - // While this is located in `nll::typeck` this error is not - // an NLL error, it's a required check to prevent creation - // of unsized rvalues in a call expression. - self.tcx().sess.emit_err(MoveUnsized { ty, span }); + if !ty.is_scalable_simd() { + if self.reported_errors.replace((ty, span)).is_none() { + // While this is located in `nll::typeck` this error is not + // an NLL error, it's a required check to prevent creation + // of unsized rvalues in a call expression. + self.tcx().sess.emit_err(MoveUnsized { ty, span }); + } } } } diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs index f601cd95f2a68..d7d972906f72b 100644 --- a/compiler/rustc_codegen_gcc/src/abi.rs +++ b/compiler/rustc_codegen_gcc/src/abi.rs @@ -94,6 +94,7 @@ impl GccType for Reg { } }, RegKind::Vector => unimplemented!(), //cx.type_vector(cx.type_i8(), self.size.bytes()), + RegKind::ScalableVector => unimplemented!(), } } } diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index ba1cae03f3e41..8a740921ee927 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -274,7 +274,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> { let layout = self.layout_of(tp_ty).layout; let _use_integer_compare = match layout.abi() { Scalar(_) | ScalarPair(_, _) => true, - Uninhabited | Vector { .. } => false, + Uninhabited | Vector { .. } | ScalableVector { .. } => false, Aggregate { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), diff --git a/compiler/rustc_codegen_gcc/src/type_of.rs b/compiler/rustc_codegen_gcc/src/type_of.rs index 479a814788a54..9496faae00c10 100644 --- a/compiler/rustc_codegen_gcc/src/type_of.rs +++ b/compiler/rustc_codegen_gcc/src/type_of.rs @@ -80,6 +80,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout false, ); } + Abi::ScalableVector { .. } => todo!(), Abi::Uninhabited | Abi::Aggregate { .. } => {} } @@ -158,7 +159,7 @@ pub trait LayoutGccExt<'tcx> { impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { fn is_gcc_immediate(&self) -> bool { match self.abi { - Abi::Scalar(_) | Abi::Vector { .. } => true, + Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => true, Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, } } @@ -166,7 +167,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> { fn is_gcc_scalar_pair(&self) -> bool { match self.abi { Abi::ScalarPair(..) => true, - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Aggregate { .. } => false, } } diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index b5f53f5183835..2ee60314821e3 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -124,6 +124,7 @@ impl LlvmType for Reg { _ => bug!("unsupported float: {:?}", self), }, RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), + RegKind::ScalableVector => cx.type_scalable_vector(cx.type_i8(), 16), } } } diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 7b259055d40b5..3f06d4da42507 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -489,7 +489,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { #[instrument(level = "trace", skip(self))] fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { - assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); + assert_eq!( + place.llextra.is_some(), + place.layout.is_unsized() && !place.layout.is_runtime_sized() + ); if place.layout.is_zst() { return OperandRef::zero_sized(place.layout); diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index cf78fc56b498c..adac834ecede9 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -1006,6 +1006,7 @@ fn build_struct_type_di_node<'ll, 'tcx>( Cow::Borrowed(f.name.as_str()) }; let field_layout = struct_type_and_layout.field(cx, i); + build_field_di_node( cx, owner, diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index cc7e78b9c62bf..3eca5a868be6a 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -302,6 +302,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { let use_integer_compare = match layout.abi() { Scalar(_) | ScalarPair(_, _) => true, Uninhabited | Vector { .. } => false, + ScalableVector { .. } => { + tcx.sess.emit_err(InvalidMonomorphization::NonScalableType { + span, + name: sym::raw_eq, + ty: tp_ty, + }); + return; + } Aggregate { .. } => { // For rusty ABIs, small aggregates are actually passed // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), @@ -983,6 +991,19 @@ fn generic_simd_intrinsic<'ll, 'tcx>( return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } + if name == sym::simd_reinterpret { + require_simd!(ret_ty, SimdReturn); + + use rustc_codegen_ssa::mir::operand::OperandValue; + return Ok(match args[0].val { + OperandValue::Ref(val, _, _) | OperandValue::Immediate(val) => { + bx.bitcast(val, llret_ty) + } + OperandValue::ZeroSized => bx.const_undef(llret_ty), + OperandValue::Pair(_, _) => todo!(), + }); + } + // every intrinsic below takes a SIMD vector as its first argument let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput); let in_ty = arg_tys[0]; @@ -1176,12 +1197,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>( InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } ); match m_elem_ty.kind() { - ty::Int(_) => {} + ty::Int(_) | ty::Bool => {} _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), } // truncate the mask to a vector of i1s let i1 = bx.type_i1(); - let i1xn = bx.type_vector(i1, m_len as u64); + let i1xn = if arg_tys[1].is_scalable_simd() { + bx.type_scalable_vector(i1, m_len as u64) + } else { + bx.type_vector(i1, m_len as u64) + }; let m_i1s = bx.trunc(args[0].immediate(), i1xn); return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); } @@ -1952,6 +1977,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( out_elem }); } + macro_rules! arith_binary { ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { $(if name == sym::$name { diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 7fc02a95be0a9..dde5d3b72e1ea 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -882,6 +882,7 @@ extern "C" { pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + pub fn LLVMScalableVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; pub fn LLVMGetElementType(Ty: &Type) -> &Type; pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 06b7703672fe8..afdda1b05eaa3 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -69,6 +69,10 @@ impl<'ll> CodegenCx<'ll, '_> { unsafe { llvm::LLVMVectorType(ty, len as c_uint) } } + pub(crate) fn type_scalable_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMScalableVectorType(ty, len as c_uint) } + } + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { unsafe { let n_args = llvm::LLVMCountParamTypes(ty) as usize; diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs index 712b6ed533303..b0d094c8e7780 100644 --- a/compiler/rustc_codegen_llvm/src/type_of.rs +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -26,6 +26,15 @@ fn uncached_llvm_type<'a, 'tcx>( let element = layout.scalar_llvm_type_at(cx, element); return cx.type_vector(element, count); } + Abi::ScalableVector { ref element, elt } => { + let element = if element.is_bool() { + cx.type_i1() + } else { + layout.scalar_llvm_type_at(cx, *element) + }; + + return cx.type_scalable_vector(element, elt); + } Abi::ScalarPair(..) => { return cx.type_struct( &[ @@ -191,7 +200,7 @@ pub trait LayoutLlvmExt<'tcx> { impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_immediate(&self) -> bool { match self.abi { - Abi::Scalar(_) | Abi::Vector { .. } => true, + Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => true, Abi::ScalarPair(..) | Abi::Uninhabited | Abi::Aggregate { .. } => false, } } @@ -199,7 +208,11 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { fn is_llvm_scalar_pair(&self) -> bool { match self.abi { Abi::ScalarPair(..) => true, - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + Abi::Uninhabited + | Abi::Scalar(_) + | Abi::Vector { .. } + | Abi::ScalableVector { .. } + | Abi::Aggregate { .. } => false, } } diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl index 5881c6236ece6..9a7ccaae94ccc 100644 --- a/compiler/rustc_codegen_ssa/messages.ftl +++ b/compiler/rustc_codegen_ssa/messages.ftl @@ -94,6 +94,7 @@ codegen_ssa_invalid_monomorphization_mask_type = invalid monomorphization of `{$ codegen_ssa_invalid_monomorphization_mismatched_lengths = invalid monomorphization of `{$name}` intrinsic: mismatched lengths: mask length `{$m_len}` != other vector length `{$v_len}` +codegen_ssa_invalid_monomorphization_non_scalable_type = invalid monomorphization of `{$name}` intrinsic: expected non-scalable type, found scalable type `{$ty}` codegen_ssa_invalid_monomorphization_return_element = invalid monomorphization of `{$name}` intrinsic: expected return element type `{$in_elem}` (element of input `{$in_ty}`), found `{$ret_ty}` with element type `{$out_ty}` codegen_ssa_invalid_monomorphization_return_integer_type = invalid monomorphization of `{$name}` intrinsic: expected return type with integer elements, found `{$ret_ty}` with non-integer `{$out_ty}` diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs index ed6ac9f9c5da8..c64d349ddc9ac 100644 --- a/compiler/rustc_codegen_ssa/src/errors.rs +++ b/compiler/rustc_codegen_ssa/src/errors.rs @@ -1035,6 +1035,14 @@ pub enum InvalidMonomorphization<'tcx> { expected_element: Ty<'tcx>, vector_type: Ty<'tcx>, }, + + #[diag(codegen_ssa_invalid_monomorphization_non_scalable_type, code = "E0511")] + NonScalableType { + #[primary_span] + span: Span, + name: Symbol, + ty: Ty<'tcx>, + }, } pub enum ExpectedPointerMutability { diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs index 14915e816ee9b..d394e6e6b3d86 100644 --- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs +++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs @@ -370,6 +370,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { if attrs.flags.contains(CodegenFnAttrFlags::NAKED) { return; } + // FIXME: Don't spill scalable simd, this works for most of them however, + // some intermediate types can't be spilled e.g. `` + if operand.layout.ty.is_scalable_simd() { + return; + } Self::spill_operand_to_stack(*operand, name, bx) } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 0ab2b7ecd9c80..d52ddb78c3aba 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -331,7 +331,10 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { bx.store(*llval, llptr, field.align.abi); *llval = bx.load(llfield_ty, llptr, field.align.abi); } - (OperandValue::Immediate(_), Abi::Uninhabited | Abi::Aggregate { sized: false }) => { + ( + OperandValue::Immediate(_), + Abi::Uninhabited | Abi::Aggregate { sized: false } | Abi::ScalableVector { .. }, + ) => { bug!() } (OperandValue::Pair(..), _) => bug!(), diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs index eb590a45a63f2..261c0a39e6c01 100644 --- a/compiler/rustc_codegen_ssa/src/mir/place.rs +++ b/compiler/rustc_codegen_ssa/src/mir/place.rs @@ -29,7 +29,7 @@ pub struct PlaceRef<'tcx, V> { impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { - assert!(layout.is_sized()); + assert!(layout.is_runtime_sized() || !layout.is_unsized()); PlaceRef { llval, llextra: None, layout, align: layout.align.abi } } @@ -38,7 +38,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { layout: TyAndLayout<'tcx>, align: Align, ) -> PlaceRef<'tcx, V> { - assert!(layout.is_sized()); + assert!(layout.is_runtime_sized() || !layout.is_unsized()); PlaceRef { llval, llextra: None, layout, align } } @@ -56,7 +56,10 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { layout: TyAndLayout<'tcx>, align: Align, ) -> Self { - assert!(layout.is_sized(), "tried to statically allocate unsized place"); + assert!( + layout.is_runtime_sized() || !layout.is_unsized(), + "tried to statically allocate unsized place" + ); let tmp = bx.alloca(bx.cx().backend_type(layout), align); Self::new_sized_aligned(tmp, layout, align) } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 02b51dfe5bf7f..275f4e24bbd25 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -1021,6 +1021,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandValueKind::Immediate(match layout.abi { abi::Abi::Scalar(s) => s, abi::Abi::Vector { element, .. } => element, + abi::Abi::ScalableVector { element, .. } => element, x => span_bug!(self.mir.span, "Couldn't translate {x:?} as backend immediate"), }) } else if self.cx.is_backend_scalar_pair(layout) { diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs index 20f251d5c91ad..29ed64f5a31a7 100644 --- a/compiler/rustc_const_eval/src/interpret/validity.rs +++ b/compiler/rustc_const_eval/src/interpret/validity.rs @@ -868,7 +868,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> self.visit_scalar(b, b_layout)?; } } - Abi::Vector { .. } => { + Abi::Vector { .. } | Abi::ScalableVector { .. } => { // No checks here, we assume layout computation gets this right. // (This is harder to check since Miri does not represent these as `Immediate`. We // also cannot use field projections since this might be a newtype around a vector.) diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs index e9e0690f07df1..47d88abce9080 100644 --- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs +++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs @@ -106,6 +106,7 @@ fn might_permit_raw_init_lax<'tcx>( Abi::Scalar(s) => scalar_allows_raw_init(s), Abi::ScalarPair(s1, s2) => scalar_allows_raw_init(s1) && scalar_allows_raw_init(s2), Abi::Vector { element: s, count } => count == 0 || scalar_allows_raw_init(s), + Abi::ScalableVector { element, .. } => scalar_allows_raw_init(element), Abi::Aggregate { .. } => true, // Fields are checked below. }; if !valid { diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs index 6680e8875c3e3..c9348e81532b1 100644 --- a/compiler/rustc_error_codes/src/error_codes.rs +++ b/compiler/rustc_error_codes/src/error_codes.rs @@ -515,6 +515,7 @@ E0792: include_str!("./error_codes/E0792.md"), E0793: include_str!("./error_codes/E0793.md"), E0794: include_str!("./error_codes/E0794.md"), E0795: include_str!("./error_codes/E0795.md"), +E0796: include_str!("./error_codes/E0796.md"), } // Undocumented removed error codes. Note that many removed error codes are kept in the list above diff --git a/compiler/rustc_error_codes/src/error_codes/E0796.md b/compiler/rustc_error_codes/src/error_codes/E0796.md new file mode 100644 index 0000000000000..b106af7cac96c --- /dev/null +++ b/compiler/rustc_error_codes/src/error_codes/E0796.md @@ -0,0 +1,10 @@ +No value of `N` was specified for `repr(scalable(N))` + +Erroneous code example: + +```compile_fail,E0796 +#[repr(scalable)] +struct Foo { + _ty: [i32; 0], +} +``` diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index e301f0b22ef77..8875440ba9a6f 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -78,7 +78,7 @@ fn check_struct(tcx: TyCtxt<'_>, def_id: LocalDefId) { def.destructor(tcx); // force the destructor to be evaluated if def.repr().simd() { - check_simd(tcx, span, def_id); + check_simd(tcx, span, def_id, def.repr().scalable()); } check_transparent(tcx, def); @@ -831,13 +831,13 @@ fn check_impl_items_against_trait<'tcx>( } } -pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) { +pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId, is_scalable: bool) { let t = tcx.type_of(def_id).instantiate_identity(); if let ty::Adt(def, args) = t.kind() && def.is_struct() { let fields = &def.non_enum_variant().fields; - if fields.is_empty() { + if fields.is_empty() && !is_scalable { struct_span_err!(tcx.sess, sp, E0075, "SIMD vector cannot be empty").emit(); return; } @@ -855,7 +855,7 @@ pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) { Some(fields.len() as u64) }; if let Some(len) = len { - if len == 0 { + if len == 0 && !is_scalable { struct_span_err!(tcx.sess, sp, E0075, "SIMD vector cannot be empty").emit(); return; } else if len > MAX_SIMD_LANES { @@ -881,6 +881,7 @@ pub fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) { ty::Array(t, _clen) if matches!(t.kind(), ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::RawPtr(_)) => { /* struct([f32; 4]) is ok */ } + ty::Slice(_) if is_scalable => (), _ => { struct_span_err!( tcx.sess, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 7ea21b24fc821..1bca703b5b4ca 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -525,6 +525,7 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) sym::simd_insert => (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0)), sym::simd_extract => (2, 0, vec![param(0), tcx.types.u32], param(1)), sym::simd_cast + | sym::simd_reinterpret | sym::simd_as | sym::simd_cast_ptr | sym::simd_expose_addr diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs index d13ae2c2094c9..109ed7962d580 100644 --- a/compiler/rustc_hir_typeck/src/check.rs +++ b/compiler/rustc_hir_typeck/src/check.rs @@ -122,7 +122,14 @@ pub(super) fn check_fn<'a, 'tcx>( hir::FnRetTy::DefaultReturn(_) => body.value.span, hir::FnRetTy::Return(ty) => ty.span, }; - fcx.require_type_is_sized(declared_ret_ty, return_or_body_span, traits::SizedReturnType); + + if !declared_ret_ty.is_scalable_simd() { + // Unsized locals and fn params have a feature gate to allow them. Return types don't + // with scalable vectors we need that feature, for now just remove the check for testing + // purposes. + fcx.require_type_is_sized(declared_ret_ty, return_or_body_span, traits::SizedReturnType); + } + fcx.check_return_expr(body.value, false); // We insert the deferred_coroutine_interiors entry after visiting the body. diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index a9f67f984da8e..c5e22b6b1951e 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -583,7 +583,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { infer::BoundRegionConversionTime::FnCall, fn_sig.output(), ); - self.require_type_is_sized_deferred(output, expr.span, traits::SizedReturnType); + if !output.is_scalable_simd() { + // Unsized locals and fn params have a feature gate to allow them. Return types don't + // with scalable vectors we need to be able to return unsized types, for now just + // remove the check for testing purposes. + self.require_type_is_sized_deferred(output, expr.span, traits::SizedReturnType); + } } // We always require that the type provided as the value for diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs index 66a0f4ed9def6..ed03b9e79b2f2 100644 --- a/compiler/rustc_hir_typeck/src/lib.rs +++ b/compiler/rustc_hir_typeck/src/lib.rs @@ -265,7 +265,10 @@ fn typeck_with_fallback<'tcx>( for (ty, span, code) in fcx.deferred_sized_obligations.borrow_mut().drain(..) { let ty = fcx.normalize(span, ty); - fcx.require_type_is_sized(ty, span, code); + // ScalableSIMD: Justify this. + if !ty.is_scalable_simd() { + fcx.require_type_is_sized(ty, span, code); + } } fcx.select_obligations_where_possible(|_| {}); diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 1d7abcf53ea3b..8868375a67ff6 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -2049,6 +2049,7 @@ impl<'tcx> TyCtxt<'tcx> { let mut size = None; let mut max_align: Option = None; let mut min_pack: Option = None; + let mut elt: Option = None; // Generate a deterministically-derived seed from the item's path hash // to allow for cross-crate compilation to actually work @@ -2077,6 +2078,10 @@ impl<'tcx> TyCtxt<'tcx> { } attr::ReprTransparent => ReprFlags::IS_TRANSPARENT, attr::ReprSimd => ReprFlags::IS_SIMD, + attr::ReprScalable(e) => { + elt = Some(e); + ReprFlags::IS_SCALABLE + } attr::ReprInt(i) => { size = Some(match i { attr::IntType::SignedInt(x) => match x { @@ -2117,7 +2122,14 @@ impl<'tcx> TyCtxt<'tcx> { flags.insert(ReprFlags::IS_LINEAR); } - ReprOptions { int: size, align: max_align, pack: min_pack, flags, field_shuffle_seed } + ReprOptions { + int: size, + align: max_align, + pack: min_pack, + flags, + field_shuffle_seed, + scalable: elt, + } } /// Look up the name of a definition across crates. This does not look at HIR. diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index f12a512da3137..d53b97446e829 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -2372,6 +2372,14 @@ impl<'tcx> Ty<'tcx> { } } + #[inline] + pub fn is_scalable_simd(self) -> bool { + match self.kind() { + Adt(def, _) => def.repr().simd() && def.repr().scalable(), + _ => false, + } + } + pub fn sequence_element_type(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { match self.kind() { Array(ty, _) | Slice(ty) => *ty, @@ -2388,6 +2396,12 @@ impl<'tcx> Ty<'tcx> { let f0_ty = variant.fields[FieldIdx::from_u32(0)].ty(tcx, args); match f0_ty.kind() { + Array(_, _) if def.repr().scalable() => { + bug!("Scalable SIMD should be using a slice, not array"); + } + Slice(f0_elem_ty) if def.repr().scalable() => { + (def.repr().scalable.unwrap_or(0) as u64, *f0_elem_ty) + } // If the first field is an array, we assume it is the only field and its // elements are the SIMD components. Array(f0_elem_ty, f0_len) => { @@ -2874,6 +2888,10 @@ impl<'tcx> Ty<'tcx> { /// This is mostly useful for optimizations, as these are the types /// on which we can replace cloning with dereferencing. pub fn is_trivially_pure_clone_copy(self) -> bool { + if self.is_scalable_simd() { + return true; + } + match self.kind() { ty::Bool | ty::Char | ty::Never => true, diff --git a/compiler/rustc_mir_build/src/build/expr/as_operand.rs b/compiler/rustc_mir_build/src/build/expr/as_operand.rs index 744111edb84e4..13c246ec128a3 100644 --- a/compiler/rustc_mir_build/src/build/expr/as_operand.rs +++ b/compiler/rustc_mir_build/src/build/expr/as_operand.rs @@ -164,8 +164,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let param_env = this.param_env; if !ty.is_sized(tcx, param_env) { - // !sized means !copy, so this is an unsized move - assert!(!ty.is_copy_modulo_regions(tcx, param_env)); + // !sized means !copy, so this is an unsized move unless it's a scalable SIMD type. + if !ty.is_scalable_simd() { + assert!(!ty.is_copy_modulo_regions(tcx, param_env)); + } // As described above, detect the case where we are passing a value of unsized // type, and that value is coming from the deref of a box. diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs index 4910d63010c6e..2f9c93d236664 100644 --- a/compiler/rustc_passes/src/check_attr.rs +++ b/compiler/rustc_passes/src/check_attr.rs @@ -1852,6 +1852,9 @@ impl CheckAttrVisitor<'_> { continue; } } + sym::scalable => { + continue; + } sym::transparent => { is_transparent = true; match target { diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index ea80bc82bd1f0..e3c4b2cc35c31 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1338,6 +1338,7 @@ symbols! { repr_align, repr_align_enum, repr_packed, + repr_scalable, repr_simd, repr_transparent, require, @@ -1480,6 +1481,7 @@ symbols! { saturating_add, saturating_div, saturating_sub, + scalable, self_in_typedefs, self_struct_ctor, semitransparent, @@ -1548,6 +1550,7 @@ symbols! { simd_reduce_mul_unordered, simd_reduce_or, simd_reduce_xor, + simd_reinterpret, simd_rem, simd_round, simd_saturating_add, diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs index f99f6a3b72164..4655f69748a87 100644 --- a/compiler/rustc_target/src/abi/call/aarch64.rs +++ b/compiler/rustc_target/src/abi/call/aarch64.rs @@ -29,6 +29,7 @@ where RegKind::Integer => false, RegKind::Float => true, RegKind::Vector => size.bits() == 64 || size.bits() == 128, + RegKind::ScalableVector => true, }; valid_unit.then_some(Uniform { unit, total: size }) diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs index 95f6691d42aeb..7de4796488fa2 100644 --- a/compiler/rustc_target/src/abi/call/arm.rs +++ b/compiler/rustc_target/src/abi/call/arm.rs @@ -19,6 +19,7 @@ where RegKind::Integer => false, RegKind::Float => true, RegKind::Vector => size.bits() == 64 || size.bits() == 128, + RegKind::ScalableVector => true, }; valid_unit.then_some(Uniform { unit, total: size }) diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs index 647b6500c52dd..e9fcea51c42c5 100644 --- a/compiler/rustc_target/src/abi/call/loongarch.rs +++ b/compiler/rustc_target/src/abi/call/loongarch.rs @@ -76,7 +76,9 @@ where } } }, - Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), + Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Uninhabited => { + return Err(CannotUseFpConv); + } Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { FieldsShape::Primitive => { unreachable!("aggregates can't have `FieldsShape::Primitive`") diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index f7c860cf56b91..8ed80364dc2f9 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -182,6 +182,7 @@ pub enum RegKind { Integer, Float, Vector, + ScalableVector, } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] @@ -228,6 +229,7 @@ impl Reg { _ => panic!("unsupported float: {self:?}"), }, RegKind::Vector => dl.vector_align(self.size).abi, + RegKind::ScalableVector => dl.vector_align(self.size).abi, } } } @@ -384,7 +386,9 @@ impl HomogeneousAggregate { impl<'a, Ty> TyAndLayout<'a, Ty> { fn is_aggregate(&self) -> bool { match self.abi { - Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::ScalableVector { .. } => { + false + } Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, } } @@ -422,6 +426,11 @@ impl<'a, Ty> TyAndLayout<'a, Ty> { })) } + Abi::ScalableVector { .. } => Ok(HomogeneousAggregate::Homogeneous(Reg { + kind: RegKind::ScalableVector, + size: Size::from_bits(128), + })), + Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => { // Helper for computing `homogeneous_aggregate`, allowing a custom // starting offset (used below for handling variants). @@ -557,6 +566,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> { ), Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout), + Abi::ScalableVector { .. } => PassMode::Direct(ArgAttributes::new()), }; ArgAbi { layout, mode } } diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs index 2d41f77e50e1e..74aeffd614e8d 100644 --- a/compiler/rustc_target/src/abi/call/powerpc64.rs +++ b/compiler/rustc_target/src/abi/call/powerpc64.rs @@ -35,6 +35,7 @@ where RegKind::Integer => false, RegKind::Float => true, RegKind::Vector => arg.layout.size.bits() == 128, + RegKind::ScalableVector => true, }; valid_unit.then_some(Uniform { unit, total: arg.layout.size }) diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs index cbde234d34cc2..19b5198e08c67 100644 --- a/compiler/rustc_target/src/abi/call/riscv.rs +++ b/compiler/rustc_target/src/abi/call/riscv.rs @@ -82,7 +82,9 @@ where } } }, - Abi::Vector { .. } | Abi::Uninhabited => return Err(CannotUseFpConv), + Abi::Vector { .. } | Abi::ScalableVector { .. } | Abi::Uninhabited => { + return Err(CannotUseFpConv); + } Abi::ScalarPair(..) | Abi::Aggregate { .. } => match arg_layout.fields { FieldsShape::Primitive => { unreachable!("aggregates can't have `FieldsShape::Primitive`") diff --git a/compiler/rustc_target/src/abi/call/x86.rs b/compiler/rustc_target/src/abi/call/x86.rs index e9aedc3d28a1e..d5e38b1efe1b1 100644 --- a/compiler/rustc_target/src/abi/call/x86.rs +++ b/compiler/rustc_target/src/abi/call/x86.rs @@ -108,6 +108,9 @@ where } false } + Abi::ScalableVector { .. } => { + unreachable!("Scalable Vectors are unsupported on this target") + } } } diff --git a/compiler/rustc_target/src/abi/call/x86_64.rs b/compiler/rustc_target/src/abi/call/x86_64.rs index 6c34585a11b82..2fbd85ebe469e 100644 --- a/compiler/rustc_target/src/abi/call/x86_64.rs +++ b/compiler/rustc_target/src/abi/call/x86_64.rs @@ -55,7 +55,7 @@ where }, Abi::Vector { .. } => Class::Sse, - + Abi::ScalableVector { .. } => panic!("Scalable vectors not supported"), Abi::ScalarPair(..) | Abi::Aggregate { .. } => { for i in 0..layout.fields.count() { let field_off = off + layout.fields.offset(i); diff --git a/compiler/rustc_target/src/abi/call/x86_win64.rs b/compiler/rustc_target/src/abi/call/x86_win64.rs index 90de1a42bc06b..fd42f85198197 100644 --- a/compiler/rustc_target/src/abi/call/x86_win64.rs +++ b/compiler/rustc_target/src/abi/call/x86_win64.rs @@ -18,6 +18,7 @@ pub fn compute_abi_info(fn_abi: &mut FnAbi<'_, Ty>) { // FIXME(eddyb) there should be a size cap here // (probably what clang calls "illegal vectors"). } + Abi::ScalableVector { .. } => {} Abi::Scalar(_) => { if a.layout.size.bytes() > 8 { a.make_indirect(); diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 826c69ee7160b..b857520d1125d 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -406,6 +406,10 @@ fn layout_of_uncached<'tcx>( }; (*e_ty, *count, true) + } else if let ty::Slice(e_ty) = f0_ty.kind() + && def.repr().scalable() + { + (*e_ty, 1, false) } else { // First ADT field is not an array: (f0_ty, def.non_enum_variant().fields.len() as _, false) @@ -445,10 +449,19 @@ fn layout_of_uncached<'tcx>( FieldsShape::Array { stride: e_ly.size, count: e_len } }; + let abi = if def.repr().scalable() { + if let Some(elt) = def.repr().scalable { + Abi::ScalableVector { element: e_abi, elt: elt as u64 } + } else { + bug!("scalable SIMD type `{}` doesn't contain the number of elements", ty,) + } + } else { + Abi::Vector { element: e_abi, count: e_len } + }; tcx.mk_layout(LayoutS { variants: Variants::Single { index: FIRST_VARIANT }, fields, - abi: Abi::Vector { element: e_abi, count: e_len }, + abi, largest_niche: e_ly.largest_niche, size, align, @@ -480,6 +493,12 @@ fn layout_of_uncached<'tcx>( return Err(error(cx, LayoutError::Unknown(ty))); } + if def.repr().scalable() + && variants[FIRST_VARIANT].iter().all(|field| !field.0.is_zst()) + { + bug!("Fields for a Scalable vector should be a ZST"); + } + return Ok(tcx.mk_layout( cx.layout_of_union(&def.repr(), &variants) .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?, diff --git a/compiler/rustc_ty_utils/src/layout_sanity_check.rs b/compiler/rustc_ty_utils/src/layout_sanity_check.rs index 6332c614a90bb..75517cb904cf6 100644 --- a/compiler/rustc_ty_utils/src/layout_sanity_check.rs +++ b/compiler/rustc_ty_utils/src/layout_sanity_check.rs @@ -77,7 +77,7 @@ pub(super) fn sanity_check_layout<'tcx>( let Some((align, size)) = align.zip(size) else { assert_matches!( layout.layout.abi(), - Abi::Uninhabited | Abi::Aggregate { .. }, + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. }, "ABI unexpectedly missing alignment and/or size in {layout:#?}" ); return; @@ -241,7 +241,7 @@ pub(super) fn sanity_check_layout<'tcx>( assert!(align >= element.align(cx).abi); // just sanity-checking `vector_align`. // FIXME: Do some kind of check of the inner type, like for Scalar and ScalarPair. } - Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check. + Abi::Uninhabited | Abi::Aggregate { .. } | Abi::ScalableVector { .. } => {} // Nothing to check. } } diff --git a/tests/ui/thir-print/thir-tree-match.stdout b/tests/ui/thir-print/thir-tree-match.stdout index 3fc130f01765a..b90e4e37bbe12 100644 --- a/tests/ui/thir-print/thir-tree-match.stdout +++ b/tests/ui/thir-print/thir-tree-match.stdout @@ -105,7 +105,7 @@ body: did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo) variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }] flags: IS_ENUM - repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 } + repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), scalable: None, field_shuffle_seed: 3477539199540094892 } args: [] variant_index: 0 subpatterns: [ @@ -119,7 +119,7 @@ body: did: DefId(0:3 ~ thir_tree_match[fcf8]::Bar) variants: [VariantDef { def_id: DefId(0:4 ~ thir_tree_match[fcf8]::Bar::First), ctor: Some((Const, DefId(0:5 ~ thir_tree_match[fcf8]::Bar::First::{constructor#0}))), name: "First", discr: Relative(0), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:6 ~ thir_tree_match[fcf8]::Bar::Second), ctor: Some((Const, DefId(0:7 ~ thir_tree_match[fcf8]::Bar::Second::{constructor#0}))), name: "Second", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:8 ~ thir_tree_match[fcf8]::Bar::Third), ctor: Some((Const, DefId(0:9 ~ thir_tree_match[fcf8]::Bar::Third::{constructor#0}))), name: "Third", discr: Relative(2), fields: [], flags: NO_VARIANT_FLAGS }] flags: IS_ENUM - repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 10333377570083945360 } + repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), scalable: None, field_shuffle_seed: 10333377570083945360 } args: [] variant_index: 0 subpatterns: [] @@ -178,7 +178,7 @@ body: did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo) variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }] flags: IS_ENUM - repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 } + repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), scalable: None, field_shuffle_seed: 3477539199540094892 } args: [] variant_index: 0 subpatterns: [ @@ -241,7 +241,7 @@ body: did: DefId(0:10 ~ thir_tree_match[fcf8]::Foo) variants: [VariantDef { def_id: DefId(0:11 ~ thir_tree_match[fcf8]::Foo::FooOne), ctor: Some((Fn, DefId(0:12 ~ thir_tree_match[fcf8]::Foo::FooOne::{constructor#0}))), name: "FooOne", discr: Relative(0), fields: [FieldDef { did: DefId(0:13 ~ thir_tree_match[fcf8]::Foo::FooOne::0), name: "0", vis: Restricted(DefId(0:0 ~ thir_tree_match[fcf8])) }], flags: NO_VARIANT_FLAGS }, VariantDef { def_id: DefId(0:14 ~ thir_tree_match[fcf8]::Foo::FooTwo), ctor: Some((Const, DefId(0:15 ~ thir_tree_match[fcf8]::Foo::FooTwo::{constructor#0}))), name: "FooTwo", discr: Relative(1), fields: [], flags: NO_VARIANT_FLAGS }] flags: IS_ENUM - repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), field_shuffle_seed: 3477539199540094892 } + repr: ReprOptions { int: None, align: None, pack: None, flags: (empty), scalable: None, field_shuffle_seed: 3477539199540094892 } args: [] variant_index: 1 subpatterns: []