Sve: Preliminary support for agnostic VL for JIT scenarios

src/coreclr/inc/clrconfigvalues.h

@@ -285,6 +285,8 @@ CONFIG_DWORD_INFO(INTERNAL_GCUseGlobalAllocationContext, W("GCUseGlobalAllocatio
 ///
 CONFIG_DWORD_INFO(INTERNAL_JitBreakEmit, W("JitBreakEmit"), (DWORD)-1, "")
 RETAIL_CONFIG_DWORD_INFO(EXTERNAL_JitDebuggable, W("JitDebuggable"), 0, "If set, suppress JIT optimizations that make debugging code difficult")
+CONFIG_DWORD_INFO(INTERNAL_UseSveForVectorT, W("UseSveForVectorT"), 0, "Prefer SVE instructions for VectorT")
+
 #if !defined(DEBUG) && !defined(_DEBUG)
 #define INTERNAL_JitEnableNoWayAssert_Default 0
 #else

src/coreclr/inc/corhdr.h

@@ -1754,6 +1754,8 @@ typedef enum CorInfoHFAElemType : unsigned {
     CORINFO_HFA_ELEM_DOUBLE,
     CORINFO_HFA_ELEM_VECTOR64,
     CORINFO_HFA_ELEM_VECTOR128,
+    CORINFO_HFA_ELEM_VECTOR256,
+    CORINFO_HFA_ELEM_VECTOR512,
 } CorInfoHFAElemType;
 
 //

src/coreclr/inc/corinfoinstructionset.h

@@ -25,24 +25,25 @@ enum CORINFO_InstructionSet
     InstructionSet_Sha1=7,
     InstructionSet_Sha256=8,
     InstructionSet_Atomics=9,
-    InstructionSet_Vector64=10,
-    InstructionSet_Vector128=11,
-    InstructionSet_Dczva=12,
-    InstructionSet_Rcpc=13,
-    InstructionSet_VectorT128=14,
-    InstructionSet_Rcpc2=15,
-    InstructionSet_Sve=16,
-    InstructionSet_Sve2=17,
-    InstructionSet_ArmBase_Arm64=18,
-    InstructionSet_AdvSimd_Arm64=19,
-    InstructionSet_Aes_Arm64=20,
-    InstructionSet_Crc32_Arm64=21,
-    InstructionSet_Dp_Arm64=22,
-    InstructionSet_Rdm_Arm64=23,
-    InstructionSet_Sha1_Arm64=24,
-    InstructionSet_Sha256_Arm64=25,
-    InstructionSet_Sve_Arm64=26,
-    InstructionSet_Sve2_Arm64=27,
+    InstructionSet_Vector=10,
+    InstructionSet_Vector64=11,
+    InstructionSet_Vector128=12,
+    InstructionSet_Dczva=13,
+    InstructionSet_Rcpc=14,
+    InstructionSet_VectorT128=15,
+    InstructionSet_Rcpc2=16,
+    InstructionSet_Sve=17,
+    InstructionSet_Sve2=18,
+    InstructionSet_ArmBase_Arm64=19,
+    InstructionSet_AdvSimd_Arm64=20,
+    InstructionSet_Aes_Arm64=21,
+    InstructionSet_Crc32_Arm64=22,
+    InstructionSet_Dp_Arm64=23,
+    InstructionSet_Rdm_Arm64=24,
+    InstructionSet_Sha1_Arm64=25,
+    InstructionSet_Sha256_Arm64=26,
+    InstructionSet_Sve_Arm64=27,
+    InstructionSet_Sve2_Arm64=28,
 #endif // TARGET_ARM64
 #ifdef TARGET_RISCV64
     InstructionSet_RiscV64Base=1,
@@ -459,6 +460,8 @@ inline CORINFO_InstructionSetFlags EnsureInstructionSetFlagsAreValid(CORINFO_Ins
             resultflags.RemoveInstructionSet(InstructionSet_Sve);
         if (resultflags.HasInstructionSet(InstructionSet_Sve2) && !resultflags.HasInstructionSet(InstructionSet_Sve))
             resultflags.RemoveInstructionSet(InstructionSet_Sve2);
+        if (resultflags.HasInstructionSet(InstructionSet_Vector) && !resultflags.HasInstructionSet(InstructionSet_Sve))
+            resultflags.RemoveInstructionSet(InstructionSet_Vector);
 #endif // TARGET_ARM64
 #ifdef TARGET_RISCV64
         if (resultflags.HasInstructionSet(InstructionSet_Zbb) && !resultflags.HasInstructionSet(InstructionSet_RiscV64Base))
@@ -883,6 +886,8 @@ inline const char *InstructionSetToString(CORINFO_InstructionSet instructionSet)
             return "Sha256_Arm64";
         case InstructionSet_Atomics :
             return "Atomics";
+        case InstructionSet_Vector :
+            return "Vector";
         case InstructionSet_Vector64 :
             return "Vector64";
         case InstructionSet_Vector128 :

src/coreclr/inc/corjit.h

@@ -438,6 +438,8 @@ class ICorJitInfo : public ICorDynamicInfo
     //
     virtual uint32_t getExpectedTargetArchitecture() = 0;
 
+    virtual uint32_t getTargetVectorLength() = 0;
+
     // Fetches extended flags for a particular compilation instance. Returns
     // the number of bytes written to the provided buffer.
     virtual uint32_t getJitFlags(

src/coreclr/inc/icorjitinfoimpl_generated.h

@@ -736,6 +736,8 @@ uint16_t getRelocTypeHint(
 
 uint32_t getExpectedTargetArchitecture() override;
 
+uint32_t getTargetVectorLength() override;
+
 uint32_t getJitFlags(
           CORJIT_FLAGS* flags,
           uint32_t sizeInBytes) override;

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* bffedb4e-ed47-4df3-8156-7ad8fc8521f1 */
-    0xbffedb4e,
-    0xed47,
-    0x4df3,
-    {0x81, 0x56, 0x7a, 0xd8, 0xfc, 0x85, 0x21, 0xf1}
+constexpr GUID JITEEVersionIdentifier = { /* 49287d16-74bd-42e9-9d47-132d7a5f67eb */
+    0x49287d16,
+    0x74bd,
+    0x42e9,
+    {0x9d, 0x47, 0x13, 0x2d, 0x7a, 0x5f, 0x67, 0xeb}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/jit/ICorJitInfo_names_generated.h

@@ -178,6 +178,7 @@ DEF_CLR_API(recordCallSite)
 DEF_CLR_API(recordRelocation)
 DEF_CLR_API(getRelocTypeHint)
 DEF_CLR_API(getExpectedTargetArchitecture)
+DEF_CLR_API(getTargetVectorLength)
 DEF_CLR_API(getJitFlags)
 DEF_CLR_API(getSpecialCopyHelper)
 

src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp

@@ -1723,6 +1723,14 @@ uint32_t WrapICorJitInfo::getExpectedTargetArchitecture()
     return temp;
 }
 
+uint32_t WrapICorJitInfo::getTargetVectorLength()
+{
+    API_ENTER(getTargetVectorLength);
+    uint32_t temp = wrapHnd->getTargetVectorLength();
+    API_LEAVE(getTargetVectorLength);
+    return temp;
+}
+
 uint32_t WrapICorJitInfo::getJitFlags(
           CORJIT_FLAGS* flags,
           uint32_t sizeInBytes)

src/coreclr/jit/abi.cpp

@@ -123,7 +123,15 @@ var_types ABIPassingSegment::GetRegisterType() const
 #ifdef FEATURE_SIMD
             case 16:
                 return TYP_SIMD16;
-#endif
+#ifdef TARGET_ARM64
+            case 32:
+                assert(Compiler::SizeMatchesVectorTLength(Size));
+                return TYP_SIMD32;
+            case 64:
+                assert(Compiler::SizeMatchesVectorTLength(Size));
+                return TYP_SIMD64;
+#endif // TARGET_ARM64
+#endif // FEATURE_SIMD
             default:
                 assert(!"Unexpected size for floating point register");
                 return TYP_UNDEF;

src/coreclr/jit/assertionprop.cpp

@@ -296,6 +296,8 @@ bool IntegralRange::Contains(int64_t value) const
                     // Example: IntCns = 42 gives [0..127] with a non -precise range, [42,42] with a precise range.
                     return {SymbolicIntegerValue::Zero, SymbolicIntegerValue::ByteMax};
 #elif defined(TARGET_ARM64)
+                case NI_Vector_op_Equality:
+                case NI_Vector_op_Inequality:
                 case NI_Vector64_op_Equality:
                 case NI_Vector64_op_Inequality:
                 case NI_Vector128_op_Equality:
@@ -2995,8 +2997,7 @@ GenTree* Compiler::optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, G
             conValTree = vecCon;
             break;
         }
-
-#if defined(TARGET_XARCH)
+#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
         case TYP_SIMD32:
         {
             simd32_t value = vnStore->ConstantValue<simd32_t>(vnCns);
@@ -3020,7 +3021,7 @@ GenTree* Compiler::optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, G
         }
         break;
 
-#endif // TARGET_XARCH
+#endif // TARGET_XARCH || TARGET_ARM64
 #endif // FEATURE_SIMD
 
 #if defined(FEATURE_MASKED_HW_INTRINSICS)

src/coreclr/jit/codegenarm64.cpp

@@ -2329,6 +2329,92 @@ void CodeGen::genSetRegToConst(regNumber targetReg, var_types targetType, GenTre
                     }
                     break;
                 }
+                case TYP_SIMD32:
+                {
+                    // Use scalable registers
+                    if (vecCon->IsAllBitsSet())
+                    {
+                        // Use Scalable_B because for Ones, it doesn't matter.
+                        emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, -1, INS_OPTS_SCALABLE_B);
+                    }
+                    else if (vecCon->IsZero())
+                    {
+                        // Use Scalable_B because for Zero, it doesn't matter.
+                        emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, 0, INS_OPTS_SCALABLE_B);
+                    }
+                    else
+                    {
+                        simd32_t val = vecCon->gtSimd32Val;
+                        if (ElementsAreSame(val.i8, 32))
+                        {
+                            emit->emitIns_R_I(INS_sve_dup, EA_SCALABLE, targetReg, val.i8[0], INS_OPTS_SCALABLE_B);
+                        }
+                        else if (ElementsAreSame(val.i16, 16))
+                        {
+                            emit->emitIns_R_I(INS_sve_dup, EA_SCALABLE, targetReg, val.i16[0], INS_OPTS_SCALABLE_H);
+                        }
+                        else if (ElementsAreSame(val.i32, 8))
+                        {
+                            emit->emitIns_R_I(INS_sve_dup, EA_SCALABLE, targetReg, val.i32[0], INS_OPTS_SCALABLE_S);
+                        }
+                        else
+                        {
+                            // Get a temp integer register to compute long address.
+                            regNumber            addrReg = internalRegisters.GetSingle(tree);
+                            CORINFO_FIELD_HANDLE hnd;
+                            hnd = emit->emitSimdConst(&vecCon->gtSimdVal, emitTypeSize(tree->TypeGet()));
+                            emit->emitIns_R_C(INS_sve_ldr, attr, targetReg, addrReg, hnd, 0);
+                            // emit->emitIns_R_C(INS_adr, EA_8BYTE, addrReg, REG_NA, hnd, 0);
+                            // emit->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, targetReg, REG_P1, addrReg, 0,
+                            // INS_OPTS_SCALABLE_B);
+                        }
+                    }
+                    break;
+                }
+                case TYP_SIMD64:
+                {
+                    // Use scalable registers
+                    if (vecCon->IsAllBitsSet())
+                    {
+                        // Use Scalable_B because for Ones, it doesn't matter.
+                        emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, -1, INS_OPTS_SCALABLE_B);
+                    }
+                    else if (vecCon->IsZero())
+                    {
+                        // Use Scalable_B because for Zero, it doesn't matter.
+                        emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, 0, INS_OPTS_SCALABLE_B);
+                    }
+                    else
+                    {
+                        simd64_t val = vecCon->gtSimd64Val;
+                        if (ElementsAreSame(val.i32, 16) && emitter::isValidSimm_MultipleOf<8, 256>(val.i32[0]))
+                        {
+                            emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, val.i32[0], INS_OPTS_SCALABLE_S,
+                                              INS_SCALABLE_OPTS_IMM_BITMASK);
+                        }
+                        else if (ElementsAreSame(val.i16, 32) && emitter::isValidSimm_MultipleOf<8, 256>(val.i16[0]))
+                        {
+                            emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, val.i16[0], INS_OPTS_SCALABLE_H,
+                                              INS_SCALABLE_OPTS_IMM_BITMASK);
+                        }
+                        else if (ElementsAreSame(val.i8, 64) && emitter::isValidSimm<8>(val.i8[0]))
+                        {
+                            emit->emitIns_R_I(INS_sve_mov, EA_SCALABLE, targetReg, val.i8[0], INS_OPTS_SCALABLE_B,
+                                              INS_SCALABLE_OPTS_IMM_BITMASK);
+                        }
+                        else
+                        {
+                            // Get a temp integer register to compute long address.
+                            regNumber            addrReg = internalRegisters.GetSingle(tree);
+                            CORINFO_FIELD_HANDLE hnd;
+                            simd64_t             constValue;
+                            memcpy(&constValue, &vecCon->gtSimdVal, sizeof(simd64_t));
+                            hnd = emit->emitSimdConst(&vecCon->gtSimdVal, emitTypeSize(tree->TypeGet()));
+                            emit->emitIns_R_C(INS_sve_ldr, attr, targetReg, addrReg, hnd, 0);
+                        }
+                    }
+                    break;
+                }
 
                 default:
                 {
@@ -2955,7 +3041,16 @@ void CodeGen::genSimpleReturn(GenTree* treeNode)
         }
     }
     emitAttr attr = emitActualTypeSize(targetType);
-    GetEmitter()->emitIns_Mov(INS_mov, attr, retReg, op1->GetRegNum(), /* canSkip */ !movRequired);
+    if (attr == EA_SCALABLE)
+    {
+        // TODO-VL: Should we check the baseType or it doesn't matter because it is just reg->reg move
+        GetEmitter()->emitIns_Mov(INS_sve_mov, attr, retReg, op1->GetRegNum(), /* canSkip */ !movRequired,
+                                  INS_OPTS_SCALABLE_Q);
+    }
+    else
+    {
+        GetEmitter()->emitIns_Mov(INS_mov, attr, retReg, op1->GetRegNum(), /* canSkip */ !movRequired);
+    }
 }
 
 /***********************************************************************************************
@@ -5248,14 +5343,28 @@ void CodeGen::genSimdUpperSave(GenTreeIntrinsic* node)
 
     GenTreeLclVar* lclNode = op1->AsLclVar();
     LclVarDsc*     varDsc  = compiler->lvaGetDesc(lclNode);
-    assert(emitTypeSize(varDsc->GetRegisterType(lclNode)) == 16);
-
-    regNumber tgtReg = node->GetRegNum();
-    assert(tgtReg != REG_NA);
+    unsigned       varSize = emitTypeSize(varDsc->GetRegisterType(lclNode));
+    assert((varSize == 16) || (Compiler::SizeMatchesVectorTLength(varSize)));
 
     regNumber op1Reg = genConsumeReg(op1);
     assert(op1Reg != REG_NA);
 
+    regNumber tgtReg = node->GetRegNum();
+#ifdef TARGET_ARM64
+    // TODO-VL: Write a helper to do this check for LclVars*, GenTree*, etc.
+    if (Compiler::UseSveForType(op1->TypeGet()))
+    {
+        // Until we custom ABI for SVE, we will just store entire contents of Z* registers
+        // on stack. If we don't do it, we will need multiple free registers to save the
+        // contents of everything but lower 8-bytes.
+        assert(tgtReg == REG_NA);
+
+        GetEmitter()->emitIns_S_R(INS_sve_str, EA_SCALABLE, op1Reg, lclNode->GetLclNum(), 0);
+        return;
+    }
+#endif // TARGET_ARM64
+    assert(tgtReg != REG_NA);
+
     GetEmitter()->emitIns_R_R_I_I(INS_mov, EA_8BYTE, tgtReg, op1Reg, 0, 1);
 
     if ((node->gtFlags & GTF_SPILL) != 0)
@@ -5304,10 +5413,12 @@ void CodeGen::genSimdUpperRestore(GenTreeIntrinsic* node)
 
     GenTreeLclVar* lclNode = op1->AsLclVar();
     LclVarDsc*     varDsc  = compiler->lvaGetDesc(lclNode);
-    assert(emitTypeSize(varDsc->GetRegisterType(lclNode)) == 16);
+
+    unsigned varSize = emitTypeSize(varDsc->GetRegisterType(lclNode));
+    assert((varSize == 16) || (Compiler::SizeMatchesVectorTLength(varSize)));
 
     regNumber srcReg = node->GetRegNum();
-    assert(srcReg != REG_NA);
+    assert((srcReg != REG_NA) || (Compiler::UseSveForType(node->TypeGet())));
 
     regNumber lclVarReg = genConsumeReg(lclNode);
     assert(lclVarReg != REG_NA);
@@ -5319,6 +5430,19 @@ void CodeGen::genSimdUpperRestore(GenTreeIntrinsic* node)
         // The localVar must have a stack home.
         assert(varDsc->lvOnFrame);
 
+#ifdef TARGET_ARM64
+        // TODO-VL: Write a helper to do this check for LclVars*, GenTree*, etc.
+        if (Compiler::UseSveForType(op1->TypeGet()))
+        {
+            // Until we custom ABI for SVE, we will just store entire contents of Z* registers
+            // on stack. If we don't do it, we will need multiple free registers to save the
+            // contents of everything but lower 8-bytes.
+
+            GetEmitter()->emitIns_R_S(INS_sve_ldr, EA_SCALABLE, lclVarReg, varNum, 0);
+            return;
+        }
+#endif // TARGET_ARM64
+
         // We will load this from the upper 8 bytes of this localVar's home.
         int offset = 8;
 

src/coreclr/jit/codegencommon.cpp

@@ -3225,8 +3225,15 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed)
             busyRegs |= genRegMask(node->copiedReg);
 
             instruction ins = ins_Copy(node->reg, copyType);
+#ifdef TARGET_ARM64
+            insOpts opts = Compiler::UseSveForType(copyType) ? INS_OPTS_SCALABLE_D : INS_OPTS_NONE;
+            GetEmitter()->emitIns_Mov(ins, emitActualTypeSize(copyType), node->copiedReg, node->reg,
+                                      /* canSkip */ false, opts);
+#else
             GetEmitter()->emitIns_Mov(ins, emitActualTypeSize(copyType), node->copiedReg, node->reg,
                                       /* canSkip */ false);
+#endif
+
             if (node->copiedReg == initReg)
             {
                 *initRegStillZeroed = false;
@@ -3243,8 +3250,15 @@ void CodeGen::genHomeRegisterParams(regNumber initReg, bool* initRegStillZeroed)
 
             regNumber   sourceReg = edge->from->copiedReg != REG_NA ? edge->from->copiedReg : edge->from->reg;
             instruction ins       = ins_Copy(sourceReg, genActualType(edge->type));
+#ifdef TARGET_ARM64
+            insOpts opts = Compiler::UseSveForType(edge->type) ? INS_OPTS_SCALABLE_D : INS_OPTS_NONE;
+            GetEmitter()->emitIns_Mov(ins, emitActualTypeSize(edge->type), node->reg, sourceReg,
+                                      /* canSkip */ true, opts);
+#else
             GetEmitter()->emitIns_Mov(ins, emitActualTypeSize(edge->type), node->reg, sourceReg,
                                       /* canSkip */ true);
+#endif
+
             break;
         }