diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 43ddbfecf006fa..2e1a284f47d480 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -24,6 +24,24 @@ public static partial class RuntimeHelpers RuntimeTypeHandle targetTypeHandle, out int count); + [Intrinsic] + [MethodImpl(MethodImplOptions.NoInlining)] + internal static unsafe uint GetObjectHeader(object obj) + { + // We need to pin the object to access its header data + // otherwise it's a GC hole (exterior pointer). + // JIT is expected to lower this call as a single load with a + // negative offset (contained) + fixed (byte* pData = &GetRawData(obj)) + { + // 64bit: [4b padding][4b header][8b pMT][data.. + // ^ + // 32bit: [4b header][4b pMT][data.. + // ^ + return *(uint*)(pData - sizeof(nint) - sizeof(int)); + } + } + // GetObjectValue is intended to allow value classes to be manipulated as 'Object' // but have aliasing behavior of a value class. The intent is that you would use // this function just before an assignment to a variable of type 'Object'. If the @@ -125,8 +143,35 @@ public static unsafe void PrepareMethod(RuntimeMethodHandle method, RuntimeTypeH [MethodImpl(MethodImplOptions.InternalCall)] public static extern void PrepareDelegate(Delegate d); + + + private const uint BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX = 0x08000000; + private const uint BIT_SBLK_IS_HASHCODE = 0x04000000; + private const uint BITS_IS_VALID_HASHCODE = BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX | BIT_SBLK_IS_HASHCODE; + private const int HASHCODE_BITS = 26; + private const uint MASK_HASHCODE = (1u << HASHCODE_BITS) - 1u; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetHashCode(object? o) + { + // WASM doesn't yet support negative offsets for loads required + // for GetObjectHeader to be optimized in RyuJIT. + if (!OperatingSystem.IsWasi() && !OperatingSystem.IsBrowser()) + { + if (o is not null) + { + uint syncBlockValue = GetObjectHeader(o); + if ((syncBlockValue & BITS_IS_VALID_HASHCODE) == BITS_IS_VALID_HASHCODE) + { + return unchecked((int)(syncBlockValue & MASK_HASHCODE)); + } + } + } + return InternalGetHashCode(o); + } + [MethodImpl(MethodImplOptions.InternalCall)] - public static extern int GetHashCode(object? o); + private static extern int InternalGetHashCode(object? o); /// /// If a hash code has been assigned to the object, it is returned. Otherwise zero is @@ -136,8 +181,27 @@ public static unsafe void PrepareMethod(RuntimeMethodHandle method, RuntimeTypeH /// The advantage of this over is that it avoids assigning a hash /// code to the object if it does not already have one. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int TryGetHashCode(object o) + { + // WASM doesn't yet support negative offsets for loads required + // for GetObjectHeader to be optimized in RyuJIT. + if (!OperatingSystem.IsWasi() && !OperatingSystem.IsBrowser()) + { + if (o is not null) + { + uint syncBlockValue = GetObjectHeader(o); + if ((syncBlockValue & BITS_IS_VALID_HASHCODE) == BITS_IS_VALID_HASHCODE) + { + return unchecked((int)(syncBlockValue & MASK_HASHCODE)); + } + } + } + return InternalTryGetHashCode(o); + } + [MethodImpl(MethodImplOptions.InternalCall)] - internal static extern int TryGetHashCode(object o); + private static extern int InternalTryGetHashCode(object? o); public static new unsafe bool Equals(object? o1, object? o2) { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.cpp b/src/coreclr/classlibnative/bcltype/objectnative.cpp index afbda5fad99167..07c4677bc2d25d 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.cpp +++ b/src/coreclr/classlibnative/bcltype/objectnative.cpp @@ -24,7 +24,7 @@ NOINLINE static INT32 GetHashCodeHelper(OBJECTREF objRef) { DWORD idx = 0; - FC_INNER_PROLOG(ObjectNative::GetHashCode); + FC_INNER_PROLOG(ObjectNative::InternalGetHashCode); HELPER_METHOD_FRAME_BEGIN_RET_ATTRIB_1(Frame::FRAME_ATTR_EXACT_DEPTH|Frame::FRAME_ATTR_CAPTURE_DEPTH_2, objRef); @@ -37,7 +37,7 @@ NOINLINE static INT32 GetHashCodeHelper(OBJECTREF objRef) // Note that we obtain a sync block index without actually building a sync block. // That's because a lot of objects are hashed, without requiring support for -FCIMPL1(INT32, ObjectNative::GetHashCode, Object* obj) { +FCIMPL1(INT32, ObjectNative::InternalGetHashCode, Object* obj) { CONTRACTL { @@ -82,7 +82,7 @@ FCIMPL1(INT32, ObjectNative::GetHashCode, Object* obj) { } FCIMPLEND -FCIMPL1(INT32, ObjectNative::TryGetHashCode, Object* obj) { +FCIMPL1(INT32, ObjectNative::InternalTryGetHashCode, Object* obj) { CONTRACTL { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.h b/src/coreclr/classlibnative/bcltype/objectnative.h index 418fd2561d7cc8..52d938a73f38f6 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.h +++ b/src/coreclr/classlibnative/bcltype/objectnative.h @@ -25,8 +25,8 @@ class ObjectNative { public: - static FCDECL1(INT32, GetHashCode, Object* vThisRef); - static FCDECL1(INT32, TryGetHashCode, Object* vThisRef); + static FCDECL1(INT32, InternalGetHashCode, Object* vThisRef); + static FCDECL1(INT32, InternalTryGetHashCode, Object* vThisRef); static FCDECL2(FC_BOOL_RET, ContentEquals, Object *pThisRef, Object *pCompareRef); static FCDECL1(Object*, GetClass, Object* pThis); static FCDECL1(FC_BOOL_RET, IsLockHeld, Object* pThisUNSAFE); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index d015332a76d8b7..09d8e610c47809 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -4688,6 +4688,8 @@ void CodeGen::genCodeForMulLong(GenTreeOp* mul) // void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) { + assert((lea->gtDebugFlags & GTF_DEBUG_LEA_EXTERIOR_PTR) == 0); + genConsumeOperands(lea); emitter* emit = GetEmitter(); emitAttr size = emitTypeSize(lea); diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index fdb99f1c29f5c9..468a438c325877 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -7216,6 +7216,8 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) // void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) { + assert((lea->gtDebugFlags & GTF_DEBUG_LEA_EXTERIOR_PTR) == 0); + genConsumeOperands(lea); emitter* emit = GetEmitter(); emitAttr size = emitTypeSize(lea); diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index 546ba7b3180899..09506680357cfb 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -7266,6 +7266,8 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) // void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) { + assert((lea->gtDebugFlags & GTF_DEBUG_LEA_EXTERIOR_PTR) == 0); + genConsumeOperands(lea); emitter* emit = GetEmitter(); emitAttr size = emitTypeSize(lea); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 4901ee9e3aa086..88aa2c2476d487 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -6787,6 +6787,8 @@ void CodeGen::genJmpMethod(GenTree* jmp) // produce code for a GT_LEA subnode void CodeGen::genLeaInstruction(GenTreeAddrMode* lea) { + assert((lea->gtDebugFlags & GTF_DEBUG_LEA_EXTERIOR_PTR) == 0); + emitAttr size = emitTypeSize(lea); genConsumeOperands(lea); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index c6369121809520..3ca16d6f51dacf 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -623,6 +623,7 @@ enum GenTreeDebugFlags : unsigned int GTF_DEBUG_VAR_CSE_REF = 0x00800000, // GT_LCL_VAR -- This is a CSE LCL_VAR node GTF_DEBUG_CAST_DONT_FOLD = 0x00400000, // GT_CAST -- Try to prevent this cast from being folded + GTF_DEBUG_LEA_EXTERIOR_PTR = 0x00200000, // GT_LEA -- It has to be expanded with offset "contained" }; inline constexpr GenTreeDebugFlags operator ~(GenTreeDebugFlags a) diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index 7fba06f2b7b3c0..ee9ea027d188c3 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -3122,6 +3122,9 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, // Not expanding this can lead to noticeable allocations in T0 case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: + // This is expanded into a single load instruction + case NI_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectHeader: + // We need these to be able to fold "typeof(...) == typeof(...)" case NI_System_Type_GetTypeFromHandle: case NI_System_Type_op_Equality: @@ -3312,6 +3315,13 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, break; } + case NI_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectHeader: + { + // It is expanded into a single load in lowering + isSpecial = true; + break; + } + case NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan: { retNode = impCreateSpanIntrinsic(sig); @@ -9633,6 +9643,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan; } + else if (strcmp(methodName, "GetObjectHeader") == 0) + { + result = NI_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectHeader; + } else if (strcmp(methodName, "InitializeArray") == 0) { result = NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray; diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index cb8eb5d6a97a0d..daf5a218fe3536 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -2484,6 +2484,61 @@ bool Lowering::LowerCallMemcmp(GenTreeCall* call, GenTree** next) return false; } +//------------------------------------------------------------------------ +// LowerCallGetObjectHeader: Replace RuntimeHelpers.GetObjectHeader with a simple +// load with a negative offset (contained) +// +// Arguments: +// tree - GenTreeCall node to expand as a load +// next - [out] Next node to lower if this function returns true +// +// Return Value: +// false if no changes were made +// +bool Lowering::LowerCallGetObjectHeader(GenTreeCall* call, GenTree** next) +{ + if (comp->info.compHasNextCallRetAddr) + { + JITDUMP("compHasNextCallRetAddr is true - bail out.\n") + return false; + } + + GenTree* objNode = call->gtArgs.GetUserArgByIndex(0)->GetNode(); + + // Access object's header using LEA with a negative offset. This LEA has to be lowered into a single + // load instruction with an addressing mode to avoid introducing a GC hole. + const ssize_t offset = -4; + GenTree* lea = new (comp, GT_LEA) GenTreeAddrMode(TYP_I_IMPL, objNode, nullptr, 1, offset); + INDEBUG(lea->gtDebugFlags |= GTF_DEBUG_LEA_EXTERIOR_PTR); + GenTreeIndir* result = comp->gtNewIndir(TYP_INT, lea); + lea->SetContained(); + + LIR::Use use; + if (BlockRange().TryGetUse(call, &use)) + { + use.ReplaceWith(result); + } + else + { + result->SetUnusedValue(); + } + + BlockRange().InsertAfter(objNode, lea, result); + BlockRange().Remove(call); + + // Remove all non-user args (e.g. r2r cell) + for (CallArg& arg : call->gtArgs.Args()) + { + if (!arg.IsUserArg()) + { + arg.GetNode()->SetUnusedValue(); + } + } + + *next = result->gtNext; + return true; +} + // do lowering steps for a call // this includes: // - adding the placement nodes (either stack or register variety) for arguments @@ -2538,6 +2593,16 @@ GenTree* Lowering::LowerCall(GenTree* node) } break; +// Target must guarantee that LEA(base, -12 or -8) is lowered as a single load with a contained imm offset +#if defined(TARGET_XARCH) || defined(TARGET_ARMARCH) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) + case NI_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectHeader: + if (LowerCallGetObjectHeader(call, &nextNode)) + { + return nextNode; + } + break; +#endif + default: break; } diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 055f425b664566..486bf672711afd 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -141,6 +141,7 @@ class Lowering final : public Phase bool LowerCallMemmove(GenTreeCall* call, GenTree** next); bool LowerCallMemcmp(GenTreeCall* call, GenTree** next); bool LowerCallMemset(GenTreeCall* call, GenTree** next); + bool LowerCallGetObjectHeader(GenTreeCall* call, GenTree** next); void LowerCFGCall(GenTreeCall* call); void MoveCFGCallArgs(GenTreeCall* call); void MoveCFGCallArg(GenTreeCall* call, GenTree* node); diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index b3eb292677d809..32b8cc61ae91b5 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -104,6 +104,7 @@ enum NamedIntrinsic : unsigned short NI_Internal_Runtime_MethodTable_Of, NI_System_Runtime_CompilerServices_RuntimeHelpers_CreateSpan, + NI_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectHeader, NI_System_Runtime_CompilerServices_RuntimeHelpers_InitializeArray, NI_System_Runtime_CompilerServices_RuntimeHelpers_IsKnownConstant, diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index dfcfbef6ba5826..6777e21905a0d0 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -445,8 +445,8 @@ FCFuncStart(gRuntimeHelpers) FCFuncElement("InitializeArray", ArrayNative::InitializeArray) FCFuncElement("GetSpanDataFrom", ArrayNative::GetSpanDataFrom) FCFuncElement("PrepareDelegate", ReflectionInvocation::PrepareDelegate) - FCFuncElement("GetHashCode", ObjectNative::GetHashCode) - FCFuncElement("TryGetHashCode", ObjectNative::TryGetHashCode) + FCFuncElement("InternalGetHashCode", ObjectNative::InternalGetHashCode) + FCFuncElement("InternalTryGetHashCode", ObjectNative::InternalTryGetHashCode) FCFuncElement("ContentEquals", ObjectNative::ContentEquals) FCFuncElement("EnsureSufficientExecutionStack", ReflectionInvocation::EnsureSufficientExecutionStack) FCFuncElement("TryEnsureSufficientExecutionStack", ReflectionInvocation::TryEnsureSufficientExecutionStack)