Skip to content

Commit

Permalink
Disallow z0-z15 in Arm64EC inline assembly
Browse files Browse the repository at this point in the history
  • Loading branch information
@taiki-e
taiki-e committed Oct 9, 2024
1 parent 750d125 commit 3307178
Show file tree
Hide file tree
Showing 7 changed files with 345 additions and 105 deletions.
23 changes: 16 additions & 7 deletions compiler/rustc_codegen_gcc/src/asm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -611,9 +611,12 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
}
// They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => "r",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg) => "w",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => "x",
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
unreachable!("clobber-only")
}
Expand Down Expand Up @@ -698,9 +701,12 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
/// the type is, as long as it is valid for the constraint code.
fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> {
match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => cx.type_i32(),
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => {
cx.type_vector(cx.type_i64(), 2)
}
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
Expand Down Expand Up @@ -863,9 +869,12 @@ fn modifier_to_gcc(
// The modifiers can be retrieved from
// https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers
match reg {
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::reg) => modifier,
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
| InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg)
| InlineAsmRegClass::Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => {
if modifier == Some('v') { None } else { modifier }
}
InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
Expand Down
88 changes: 59 additions & 29 deletions compiler/rustc_codegen_llvm/src/asm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -656,9 +656,12 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
// The constraints can be retrieved from
// https://llvm.org/docs/LangRef.html#supported-constraint-code-list
InlineAsmRegOrRegClass::RegClass(reg) => match reg {
AArch64(AArch64InlineAsmRegClass::reg) => "r",
AArch64(AArch64InlineAsmRegClass::vreg) => "w",
AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => "r",
AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg) => {
"w"
}
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => "x",
AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
Arm(ArmInlineAsmRegClass::reg) => "r",
Arm(ArmInlineAsmRegClass::sreg)
Expand Down Expand Up @@ -734,8 +737,11 @@ fn modifier_to_llvm(
// The modifiers can be retrieved from
// https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
match reg {
AArch64(AArch64InlineAsmRegClass::reg) => modifier,
AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => modifier,
AArch64(AArch64InlineAsmRegClass::vreg)
| AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => {
if modifier == Some('v') {
None
} else {
Expand Down Expand Up @@ -817,10 +823,13 @@ fn modifier_to_llvm(
fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
use InlineAsmRegClass::*;
match reg {
AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
cx.type_vector(cx.type_i64(), 2)
AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(Arm64ECInlineAsmRegClass::reg) => {
cx.type_i32()
}
AArch64(AArch64InlineAsmRegClass::vreg)
| AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16) => cx.type_vector(cx.type_i64(), 2),
AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"),
Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
Expand Down Expand Up @@ -922,17 +931,22 @@ fn llvm_fixup_input<'ll, 'tcx>(
use InlineAsmRegClass::*;
let dl = &bx.tcx.data_layout;
match (reg, layout.abi) {
(AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
(
AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I8, _) = s.primitive() {
let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
} else {
value
}
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
if s.primitive() != Primitive::Float(Float::F128) =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Scalar(s),
) if s.primitive() != Primitive::Float(Float::F128) => {
let elem_ty = llvm_asm_scalar_type(bx.cx, s);
let count = 16 / layout.size.bytes();
let vec_ty = bx.cx.type_vector(elem_ty, count);
Expand All @@ -943,9 +957,11 @@ fn llvm_fixup_input<'ll, 'tcx>(
}
bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
if layout.size.bytes() == 8 =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Vector { element, count },
) if layout.size.bytes() == 8 => {
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
let vec_ty = bx.cx.type_vector(elem_ty, count);
let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect();
Expand Down Expand Up @@ -1064,25 +1080,32 @@ fn llvm_fixup_output<'ll, 'tcx>(
) -> &'ll Value {
use InlineAsmRegClass::*;
match (reg, layout.abi) {
(AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
(
AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I8, _) = s.primitive() {
bx.extract_element(value, bx.const_i32(0))
} else {
value
}
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
if s.primitive() != Primitive::Float(Float::F128) =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Scalar(s),
) if s.primitive() != Primitive::Float(Float::F128) => {
value = bx.extract_element(value, bx.const_i32(0));
if let Primitive::Pointer(_) = s.primitive() {
value = bx.inttoptr(value, layout.llvm_type(bx.cx));
}
value
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
if layout.size.bytes() == 8 =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Vector { element, count },
) if layout.size.bytes() == 8 => {
let elem_ty = llvm_asm_scalar_type(bx.cx, element);
let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
Expand Down Expand Up @@ -1195,23 +1218,30 @@ fn llvm_fixup_output_type<'ll, 'tcx>(
) -> &'ll Type {
use InlineAsmRegClass::*;
match (reg, layout.abi) {
(AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {
(
AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(Arm64ECInlineAsmRegClass::vreg),
Abi::Scalar(s),
) => {
if let Primitive::Int(Integer::I8, _) = s.primitive() {
cx.type_vector(cx.type_i8(), 8)
} else {
layout.llvm_type(cx)
}
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s))
if s.primitive() != Primitive::Float(Float::F128) =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Scalar(s),
) if s.primitive() != Primitive::Float(Float::F128) => {
let elem_ty = llvm_asm_scalar_type(cx, s);
let count = 16 / layout.size.bytes();
cx.type_vector(elem_ty, count)
}
(AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count })
if layout.size.bytes() == 8 =>
{
(
AArch64(AArch64InlineAsmRegClass::vreg_low16)
| Arm64EC(Arm64ECInlineAsmRegClass::vreg_low16),
Abi::Vector { element, count },
) if layout.size.bytes() == 8 => {
let elem_ty = llvm_asm_scalar_type(cx, element);
cx.type_vector(elem_ty, count * 2)
}
Expand Down
91 changes: 38 additions & 53 deletions compiler/rustc_target/src/asm/aarch64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,20 +88,6 @@ fn reserved_x18(
}
}

fn restricted_for_arm64ec(
arch: InlineAsmArch,
_reloc_model: RelocModel,
_target_features: &FxIndexSet<Symbol>,
_target: &Target,
_is_clobber: bool,
) -> Result<(), &'static str> {
if arch == InlineAsmArch::Arm64EC {
Err("x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC")
} else {
Ok(())
}
}

def_regs! {
AArch64 AArch64InlineAsmReg AArch64InlineAsmRegClass {
x0: reg = ["x0", "w0"],
Expand All @@ -117,23 +103,22 @@ def_regs! {
x10: reg = ["x10", "w10"],
x11: reg = ["x11", "w11"],
x12: reg = ["x12", "w12"],
x13: reg = ["x13", "w13"] % restricted_for_arm64ec,
x14: reg = ["x14", "w14"] % restricted_for_arm64ec,
x13: reg = ["x13", "w13"],
x14: reg = ["x14", "w14"],
x15: reg = ["x15", "w15"],
x16: reg = ["x16", "w16"],
x17: reg = ["x17", "w17"],
x18: reg = ["x18", "w18"] % reserved_x18,
x20: reg = ["x20", "w20"],
x21: reg = ["x21", "w21"],
x22: reg = ["x22", "w22"],
x23: reg = ["x23", "w23"] % restricted_for_arm64ec,
x24: reg = ["x24", "w24"] % restricted_for_arm64ec,
x23: reg = ["x23", "w23"],
x24: reg = ["x24", "w24"],
x25: reg = ["x25", "w25"],
x26: reg = ["x26", "w26"],
x27: reg = ["x27", "w27"],
x28: reg = ["x28", "w28"] % restricted_for_arm64ec,
x28: reg = ["x28", "w28"],
x30: reg = ["x30", "w30", "lr", "wlr"],
// FIXME: z* cannot be used for Arm64EC
v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0", "z0"],
v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1", "z1"],
v2: vreg, vreg_low16 = ["v2", "b2", "h2", "s2", "d2", "q2", "z2"],
Expand All @@ -150,39 +135,39 @@ def_regs! {
v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13", "z13"],
v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14", "z14"],
v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15", "z15"],
v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"] % restricted_for_arm64ec,
v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"] % restricted_for_arm64ec,
v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"] % restricted_for_arm64ec,
v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"] % restricted_for_arm64ec,
v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"] % restricted_for_arm64ec,
v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"] % restricted_for_arm64ec,
v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"] % restricted_for_arm64ec,
v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"] % restricted_for_arm64ec,
v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"] % restricted_for_arm64ec,
v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"] % restricted_for_arm64ec,
v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"] % restricted_for_arm64ec,
v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"] % restricted_for_arm64ec,
v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"] % restricted_for_arm64ec,
v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"] % restricted_for_arm64ec,
v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"] % restricted_for_arm64ec,
v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"] % restricted_for_arm64ec,
p0: preg = ["p0"] % restricted_for_arm64ec,
p1: preg = ["p1"] % restricted_for_arm64ec,
p2: preg = ["p2"] % restricted_for_arm64ec,
p3: preg = ["p3"] % restricted_for_arm64ec,
p4: preg = ["p4"] % restricted_for_arm64ec,
p5: preg = ["p5"] % restricted_for_arm64ec,
p6: preg = ["p6"] % restricted_for_arm64ec,
p7: preg = ["p7"] % restricted_for_arm64ec,
p8: preg = ["p8"] % restricted_for_arm64ec,
p9: preg = ["p9"] % restricted_for_arm64ec,
p10: preg = ["p10"] % restricted_for_arm64ec,
p11: preg = ["p11"] % restricted_for_arm64ec,
p12: preg = ["p12"] % restricted_for_arm64ec,
p13: preg = ["p13"] % restricted_for_arm64ec,
p14: preg = ["p14"] % restricted_for_arm64ec,
p15: preg = ["p15"] % restricted_for_arm64ec,
ffr: preg = ["ffr"] % restricted_for_arm64ec,
v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"],
v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"],
v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"],
v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"],
v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"],
v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"],
v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"],
v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"],
v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"],
v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"],
v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"],
v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"],
v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"],
v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"],
v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"],
v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"],
p0: preg = ["p0"],
p1: preg = ["p1"],
p2: preg = ["p2"],
p3: preg = ["p3"],
p4: preg = ["p4"],
p5: preg = ["p5"],
p6: preg = ["p6"],
p7: preg = ["p7"],
p8: preg = ["p8"],
p9: preg = ["p9"],
p10: preg = ["p10"],
p11: preg = ["p11"],
p12: preg = ["p12"],
p13: preg = ["p13"],
p14: preg = ["p14"],
p15: preg = ["p15"],
ffr: preg = ["ffr"],
#error = ["x19", "w19"] =>
"x19 is used internally by LLVM and cannot be used as an operand for inline asm",
#error = ["x29", "w29", "fp", "wfp"] =>
Expand Down
Loading

0 comments on commit 3307178

Please sign in to comment.