Move unboxing helpers to managed code (#109135)

Move the unboxing helpers to managed code.

Behavior is basically identical except for the Unbox_Nullable paths, which required some investigation to find the fastest implementation. Notably, there interruptibility of managed code makes the copying/zeroing of values more difficult, but with the opportunity/requirement to specialize the codebase came a few micro-optimizations that are somewhat nice. Overall I don't expect anyone to notice the performance changes here, but since my earlier code was about 2X slower than the native implementation, I did feel the need to optimize until everything looked good.

Performance results:

| TestName | With PR | Without PR | % Speedup |
| --- | --- | --- | --- |
|                     TestJustAPrimitive| 1217.0047| 1221.203| 0.34%   |
|                         TestJustObject| 1212.6415| 1211.6437|-0.08%  |
|                     TestJust5Primitive| 1496.5304| 1522.4968|1.74%   |
|                        TestJust5Object| 1461.0507| 1488.3328|1.87%   |
|                    TestJust10Primitive| 1473.2814| 1493.5238|1.37%   |
|                       TestJust10Object| 3215.6339| 2854.6186|-11.23% |
|    TestJustAPrimitiveNullableWithValue| 2727.9085| 5182.2611|89.97%  |
|        TestJustObjectNullableWithValue| 3148.9484| 5672.2985|80.13%  |
|    TestJust5PrimitiveNullableWithValue| 5443.9232| 7795.6109|43.20%  |
|       TestJust5ObjectNullableWithValue| 6492.9071| 8095.1508|24.68%  |
|   TestJust10PrimitiveNullableWithValue| 6022.6274| 8723.572| 44.85%  |
|      TestJust10ObjectNullableWithValue| 7728.3239| 9671.1382|25.14%  |
|         TestJustAPrimitiveNullNullable| 1786.1337| 2230.0932|24.86%  |
|             TestJustObjectNullNullable| 1675.0683| 2326.0395|38.86%  |
|         TestJust5PrimitiveNullNullable| 2921.9497| 3298.4642|12.89%  |
|            TestJust5ObjectNullNullable| 3389.4043| 3615.3131|6.67%   |
|        TestJust10PrimitiveNullNullable| 3050.809|	 4054.9683|32.91%  |
|           TestJust10ObjectNullNullable| 4658.8316| 5335.0686|14.52%  |

Results are very positive, or within the margin of error in this test suite. These results were generated using a small benchmark which mostly targeted measuring the performance of the Unbox_Nullable helper, as it has the most complex and potentially slow code. Generally the impact on that helper is that the performance of the type system portion of the helper is faster, and the performance of code which actually copies the contents of a valuetype is a little better. This isn't quite a fair test of managed vs native performance though, as I took the opportunity to restructure some of the memory on `MethodTable` so that it could more easily be read in managed code, and that happened to make a fair bit of complex code become simpler and thus faster.

src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs

@@ -253,7 +253,7 @@ private static unsafe void CopyImplUnBoxEachElement(Array sourceArray, int sourc
 
                 if (pDestMT->IsNullable)
                 {
-                    RuntimeHelpers.Unbox_Nullable(ref dest, pDestMT, obj);
+                    CastHelpers.Unbox_Nullable(ref dest, pDestMT, obj);
                 }
                 else if (obj is null || RuntimeHelpers.GetMethodTable(obj) != pDestMT)
                 {
@@ -518,11 +518,16 @@ private unsafe void InternalSetValue(object? value, nint flattenedIndex)
                 if (pElementMethodTable->IsValueType)
                 {
                     ref byte offsetDataRef = ref Unsafe.Add(ref arrayDataRef, flattenedIndex * pMethodTable->ComponentSize);
-                    nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytes();
                     if (pElementMethodTable->ContainsGCPointers)
+                    {
+                        nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytesIfContainsGCPointers();
                         SpanHelpers.ClearWithReferences(ref Unsafe.As<byte, nint>(ref offsetDataRef), elementSize / (nuint)sizeof(IntPtr));
+                    }
                     else
+                    {
+                        nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytes();
                         SpanHelpers.ClearWithoutReferences(ref offsetDataRef, elementSize);
+                    }
                 }
                 else
                 {
@@ -546,17 +551,18 @@ private unsafe void InternalSetValue(object? value, nint flattenedIndex)
                 {
                     if (pElementMethodTable->IsNullable)
                     {
-                        RuntimeHelpers.Unbox_Nullable(ref offsetDataRef, pElementMethodTable, value);
+                        CastHelpers.Unbox_Nullable(ref offsetDataRef, pElementMethodTable, value);
                     }
                     else
                     {
-                        nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytes();
                         if (pElementMethodTable->ContainsGCPointers)
                         {
+                            nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytesIfContainsGCPointers();
                             Buffer.BulkMoveWithWriteBarrier(ref offsetDataRef, ref value.GetRawData(), elementSize);
                         }
                         else
                         {
+                            nuint elementSize = pElementMethodTable->GetNumInstanceFieldBytes();
                             SpanHelpers.Memmove(ref offsetDataRef, ref value.GetRawData(), elementSize);
                         }
                     }

src/coreclr/System.Private.CoreLib/src/System/Delegate.CoreCLR.cs

@@ -456,7 +456,7 @@ internal static unsafe bool InternalEqualTypes(object a, object b)
         {
             if (a.GetType() == b.GetType())
                 return true;
-
+#if FEATURE_TYPEEQUIVALENCE
             MethodTable* pMTa = RuntimeHelpers.GetMethodTable(a);
             MethodTable* pMTb = RuntimeHelpers.GetMethodTable(b);
 
@@ -472,6 +472,9 @@ internal static unsafe bool InternalEqualTypes(object a, object b)
             GC.KeepAlive(b);
 
             return ret;
+#else
+            return false;
+#endif // FEATURE_TYPEEQUIVALENCE
         }
 
         // Used by the ctor. Do not call directly.

src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/CastHelpers.cs

@@ -2,26 +2,34 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Runtime.InteropServices;
 
 namespace System.Runtime.CompilerServices
 {
     [StackTraceHidden]
     [DebuggerStepThrough]
-    internal static unsafe class CastHelpers
+    internal static unsafe partial class CastHelpers
     {
         // In coreclr the table is allocated and written to on the native side.
         internal static int[]? s_table;
 
+        [LibraryImport(RuntimeHelpers.QCall)]
+        internal static partial void ThrowInvalidCastException(void* fromTypeHnd, void* toTypeHnd);
+
+        [DoesNotReturn]
+        internal static void ThrowInvalidCastException(object fromType, void* toTypeHnd)
+        {
+            ThrowInvalidCastException(RuntimeHelpers.GetMethodTable(fromType), toTypeHnd);
+            throw null!; // Provide hint to the inliner that this method does not return
+        }
+
         [MethodImpl(MethodImplOptions.InternalCall)]
         private static extern object IsInstanceOfAny_NoCacheLookup(void* toTypeHnd, object obj);
 
         [MethodImpl(MethodImplOptions.InternalCall)]
         private static extern object ChkCastAny_NoCacheLookup(void* toTypeHnd, object obj);
 
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        private static extern ref byte Unbox_Helper(void* toTypeHnd, object obj);
-
         [MethodImpl(MethodImplOptions.InternalCall)]
         private static extern void WriteBarrier(ref object? dst, object? obj);
 
@@ -365,7 +373,7 @@ internal static unsafe class CastHelpers
         }
 
         [DebuggerHidden]
-        private static ref byte Unbox(void* toTypeHnd, object obj)
+        private static ref byte Unbox(MethodTable* toTypeHnd, object obj)
         {
             // This will throw NullReferenceException if obj is null.
             if (RuntimeHelpers.GetMethodTable(obj) == toTypeHnd)
@@ -492,5 +500,145 @@ private static unsafe void ArrayTypeCheck_Helper(object obj, void* elementType)
                 ThrowArrayMismatchException();
             }
         }
+
+        // Helpers for Unboxing
+#if FEATURE_TYPEEQUIVALENCE
+        [DebuggerHidden]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static bool AreTypesEquivalent(MethodTable* pMTa, MethodTable* pMTb)
+        {
+            if (pMTa == pMTb)
+            {
+                return true;
+            }
+
+            if (!pMTa->HasTypeEquivalence || !pMTb->HasTypeEquivalence)
+            {
+                return false;
+            }
+
+            return RuntimeHelpers.AreTypesEquivalent(pMTa, pMTb);
+        }
+#endif // FEATURE_TYPEEQUIVALENCE
+
+        [DebuggerHidden]
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static bool IsNullableForType(MethodTable* typeMT, MethodTable* boxedMT)
+        {
+            if (!typeMT->IsNullable)
+            {
+                return false;
+            }
+
+            // Normally getting the first generic argument involves checking the PerInstInfo to get the count of generic dictionaries
+            // in the hierarchy, and then doing a bit of math to find the right dictionary, but since we know this is nullable
+            // we can do a simple double deference to do the same thing.
+            Debug.Assert(typeMT->InstantiationArg0() == **typeMT->PerInstInfo);
+            MethodTable *pMTNullableArg = **typeMT->PerInstInfo;
+            if (pMTNullableArg == boxedMT)
+            {
+                return true;
+            }
+            else
+            {
+#if FEATURE_TYPEEQUIVALENCE
+                return AreTypesEquivalent(pMTNullableArg, boxedMT);
+#else
+                return false;
+#endif // FEATURE_TYPEEQUIVALENCE
+            }
+        }
+
+        [DebuggerHidden]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void Unbox_Nullable_NotIsNullableForType(ref byte destPtr, MethodTable* typeMT, object obj)
+        {
+            // Also allow true nullables to be unboxed normally.
+            // This should not happen normally, but can happen in debugger scenarios.
+            if (typeMT != RuntimeHelpers.GetMethodTable(obj))
+            {
+                CastHelpers.ThrowInvalidCastException(obj, typeMT);
+            }
+            Buffer.BulkMoveWithWriteBarrier(ref destPtr, ref RuntimeHelpers.GetRawData(obj), typeMT->GetNullableNumInstanceFieldBytes());
+        }
+
+        [DebuggerHidden]
+        internal static void Unbox_Nullable(ref byte destPtr, MethodTable* typeMT, object? obj)
+        {
+            if (obj == null)
+            {
+                if (!typeMT->ContainsGCPointers)
+                {
+                    SpanHelpers.ClearWithoutReferences(ref destPtr, typeMT->GetNullableNumInstanceFieldBytes());
+                }
+                else
+                {
+                    SpanHelpers.ClearWithReferences(ref Unsafe.As<byte, IntPtr>(ref destPtr), typeMT->GetNumInstanceFieldBytesIfContainsGCPointers() / (nuint)sizeof(IntPtr));
+                }
+            }
+            else
+            {
+                if (!IsNullableForType(typeMT, RuntimeHelpers.GetMethodTable(obj)))
+                {
+                    Unbox_Nullable_NotIsNullableForType(ref destPtr, typeMT, obj);
+                }
+                else
+                {
+                    Unsafe.As<byte, bool>(ref destPtr) = true;
+                    ref byte dst = ref Unsafe.Add(ref destPtr, typeMT->NullableValueAddrOffset);
+                    uint valueSize = typeMT->NullableValueSize;
+                    ref byte src = ref RuntimeHelpers.GetRawData(obj);
+                    if (typeMT->ContainsGCPointers)
+                        Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, valueSize);
+                    else
+                        SpanHelpers.Memmove(ref dst, ref src, valueSize);
+                }
+            }
+        }
+
+        [DebuggerHidden]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static ref byte Unbox_Helper(MethodTable* pMT1, object obj)
+        {
+            // must be a value type
+            Debug.Assert(pMT1->IsValueType);
+
+            MethodTable* pMT2 = RuntimeHelpers.GetMethodTable(obj);
+            if ((!pMT1->IsPrimitive || !pMT2->IsPrimitive ||
+                pMT1->GetPrimitiveCorElementType() != pMT2->GetPrimitiveCorElementType())
+#if FEATURE_TYPEEQUIVALENCE
+                && !AreTypesEquivalent(pMT1, pMT2)
+#endif // FEATURE_TYPEEQUIVALENCE
+                )
+            {
+                CastHelpers.ThrowInvalidCastException(obj, pMT1);
+            }
+
+            return ref RuntimeHelpers.GetRawData(obj);
+        }
+
+        [DebuggerHidden]
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void Unbox_TypeTest_Helper(MethodTable *pMT1, MethodTable *pMT2)
+        {
+            if ((!pMT1->IsPrimitive || !pMT2->IsPrimitive ||
+                pMT1->GetPrimitiveCorElementType() != pMT2->GetPrimitiveCorElementType())
+#if FEATURE_TYPEEQUIVALENCE
+                && !AreTypesEquivalent(pMT1, pMT2)
+#endif // FEATURE_TYPEEQUIVALENCE
+                )
+            {
+                CastHelpers.ThrowInvalidCastException(pMT1, pMT2);
+            }
+        }
+
+        [DebuggerHidden]
+        private static void Unbox_TypeTest(MethodTable *pMT1, MethodTable *pMT2)
+        {
+            if (pMT1 == pMT2)
+                return;
+            else
+                Unbox_TypeTest_Helper(pMT1, pMT2);
+        }
     }
 }

src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs

@@ -448,9 +448,6 @@ internal static unsafe bool ObjectHasComponentSize(object obj)
         [MethodImpl(MethodImplOptions.InternalCall)]
         internal static extern unsafe object? Box(MethodTable* methodTable, ref byte data);
 
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        internal static extern unsafe void Unbox_Nullable(ref byte destination, MethodTable* toTypeHnd, object? obj);
-
         // Given an object reference, returns its MethodTable*.
         //
         // WARNING: The caller has to ensure that MethodTable* does not get unloaded. The most robust way
@@ -706,12 +703,41 @@ internal unsafe struct MethodTable
         [FieldOffset(ElementTypeOffset)]
         public void* ElementType;
 
+        /// <summary>
+        /// The PerInstInfo is used to describe the generic arguments and dictionary of this type.
+        /// It points at a structure defined as PerInstInfo in C++, which is an array of pointers to generic
+        /// dictionaries, which then point to the actual type arguments + the contents of the generic dictionary.
+        /// The size of the PerInstInfo is defined in the negative space of that structure, and the size of the
+        /// generic dictionary is described in the DictionaryLayout of the associated canonical MethodTable.
+        /// </summary>
+        [FieldOffset(ElementTypeOffset)]
+        public MethodTable*** PerInstInfo;
+
         /// <summary>
         /// This interface map used to list out the set of interfaces. Only meaningful if InterfaceCount is non-zero.
         /// </summary>
         [FieldOffset(InterfaceMapOffset)]
         public MethodTable** InterfaceMap;
 
+        /// <summary>
+        /// This is used to hold the nullable unbox data for nullable value types.
+        /// </summary>
+        [FieldOffset(InterfaceMapOffset)]
+#if TARGET_64BIT
+        public uint NullableValueAddrOffset;
+#else
+        public byte NullableValueAddrOffset;
+#endif
+
+#if TARGET_64BIT
+        [FieldOffset(InterfaceMapOffset + 4)]
+        public uint NullableValueSize;
+#else
+        [FieldOffset(InterfaceMapOffset)]
+        private uint NullableValueSizeEncoded;
+        public uint NullableValueSize => NullableValueSizeEncoded >> 8;
+#endif
+
         // WFLAGS_LOW_ENUM
         private const uint enum_flag_GenericsMask = 0x00000030;
         private const uint enum_flag_GenericsMask_NonGeneric = 0x00000000; // no instantiation
@@ -730,6 +756,7 @@ internal unsafe struct MethodTable
         private const uint enum_flag_Category_Mask = 0x000F0000;
         private const uint enum_flag_Category_ValueType = 0x00040000;
         private const uint enum_flag_Category_Nullable = 0x00050000;
+        private const uint enum_flag_Category_IsPrimitiveMask = 0x000E0000;
         private const uint enum_flag_Category_PrimitiveValueType = 0x00060000; // sub-category of ValueType, Enum or primitive value type
         private const uint enum_flag_Category_TruePrimitive = 0x00070000; // sub-category of ValueType, Primitive (ELEMENT_TYPE_I, etc.)
         private const uint enum_flag_Category_Array = 0x00080000;
@@ -780,7 +807,9 @@ internal unsafe struct MethodTable
 
         public bool NonTrivialInterfaceCast => (Flags & enum_flag_NonTrivialInterfaceCast) != 0;
 
+#if FEATURE_TYPEEQUIVALENCE
         public bool HasTypeEquivalence => (Flags & enum_flag_HasTypeEquivalence) != 0;
+#endif // FEATURE_TYPEEQUIVALENCE
 
         public bool HasFinalizer => (Flags & enum_flag_HasFinalizer) != 0;
 
@@ -815,12 +844,13 @@ public int MultiDimensionalArrayRank
 
         public bool IsValueType => (Flags & enum_flag_Category_ValueType_Mask) == enum_flag_Category_ValueType;
 
-        public bool IsNullable => (Flags & enum_flag_Category_Mask) == enum_flag_Category_Nullable;
+
+        public bool IsNullable { [MethodImpl(MethodImplOptions.AggressiveInlining)] get { return (Flags & enum_flag_Category_Mask) == enum_flag_Category_Nullable; } }
 
         public bool IsByRefLike => (Flags & (enum_flag_HasComponentSize | enum_flag_IsByRefLike)) == enum_flag_IsByRefLike;
 
         // Warning! UNLIKE the similarly named Reflection api, this method also returns "true" for Enums.
-        public bool IsPrimitive => (Flags & enum_flag_Category_Mask) is enum_flag_Category_PrimitiveValueType or enum_flag_Category_TruePrimitive;
+        public bool IsPrimitive => (Flags & enum_flag_Category_IsPrimitiveMask) == enum_flag_Category_PrimitiveValueType;
 
         public bool IsTruePrimitive => (Flags & enum_flag_Category_Mask) is enum_flag_Category_TruePrimitive;
 
@@ -877,6 +907,27 @@ public TypeHandle GetArrayElementTypeHandle()
         /// </summary>
         [MethodImpl(MethodImplOptions.InternalCall)]
         public extern MethodTable* GetMethodTableMatchingParentClass(MethodTable* parent);
+
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public extern MethodTable* InstantiationArg0();
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public uint GetNullableNumInstanceFieldBytes()
+        {
+            Debug.Assert(IsNullable);
+            Debug.Assert((NullableValueAddrOffset + NullableValueSize) == GetNumInstanceFieldBytes());
+            return NullableValueAddrOffset + NullableValueSize;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public uint GetNumInstanceFieldBytesIfContainsGCPointers()
+        {
+            // If the type ContainsGCPointers, we can compute the size without resorting to loading the BaseSizePadding field from the EEClass
+
+            Debug.Assert(ContainsGCPointers);
+            Debug.Assert((BaseSize - (nuint)(2 * sizeof(IntPtr)) == GetNumInstanceFieldBytes()));
+            return BaseSize - (uint)(2 * sizeof(IntPtr));
+        }
     }
 
     [StructLayout(LayoutKind.Sequential)]

src/coreclr/inc/jithelpers.h

@@ -114,8 +114,8 @@
     DYNAMICJITHELPER(CORINFO_HELP_BOX,          JIT_Box,            METHOD__NIL)
     JITHELPER(CORINFO_HELP_BOX_NULLABLE,        JIT_Box,            METHOD__NIL)
     DYNAMICJITHELPER(CORINFO_HELP_UNBOX,        NULL,               METHOD__CASTHELPERS__UNBOX)
-    JITHELPER(CORINFO_HELP_UNBOX_TYPETEST,      JIT_Unbox_TypeTest, METHOD__NIL)
-    JITHELPER(CORINFO_HELP_UNBOX_NULLABLE,      JIT_Unbox_Nullable, METHOD__NIL)
+    DYNAMICJITHELPER(CORINFO_HELP_UNBOX_TYPETEST,NULL,              METHOD__CASTHELPERS__UNBOX_TYPETEST)
+    DYNAMICJITHELPER(CORINFO_HELP_UNBOX_NULLABLE,NULL,              METHOD__CASTHELPERS__UNBOX_NULLABLE)
 
     JITHELPER(CORINFO_HELP_GETREFANY,           JIT_GetRefAny,      METHOD__NIL)
     DYNAMICJITHELPER(CORINFO_HELP_ARRADDR_ST,   NULL,               METHOD__CASTHELPERS__STELEMREF)

src/coreclr/vm/JitQCallHelpers.h

@@ -21,6 +21,7 @@ class MethodDesc;
 extern "C" void * QCALLTYPE ResolveVirtualFunctionPointer(QCall::ObjectHandleOnStack obj, CORINFO_CLASS_HANDLE classHnd, CORINFO_METHOD_HANDLE methodHnd);
 extern "C" CORINFO_GENERIC_HANDLE QCALLTYPE GenericHandleWorker(MethodDesc * pMD, MethodTable * pMT, LPVOID signature, DWORD dictionaryIndexAndSlot, Module* pModule);
 extern "C" void QCALLTYPE InitClassHelper(MethodTable* pMT);
+extern "C" void QCALLTYPE ThrowInvalidCastException(CORINFO_CLASS_HANDLE pTargetType, CORINFO_CLASS_HANDLE pSourceType);
 extern "C" void QCALLTYPE GetThreadStaticsByMethodTable(QCall::ByteRefOnStack refHandle, MethodTable* pMT, bool gcStatic);
 extern "C" void QCALLTYPE GetThreadStaticsByIndex(QCall::ByteRefOnStack refHandle, uint32_t staticBlockIndex, bool gcStatic);