diff --git a/src/coreclr/jit/abi.cpp b/src/coreclr/jit/abi.cpp index 0525e18888f46b..198f523f6b6b70 100644 --- a/src/coreclr/jit/abi.cpp +++ b/src/coreclr/jit/abi.cpp @@ -154,6 +154,29 @@ var_types ABIPassingSegment::GetRegisterType() const } } +//----------------------------------------------------------------------------- +// GetRegisterType: +// Return the smallest type larger or equal to Size that most naturally +// represents the register this segment is passed in, taking into account the +// GC info of the specified layout. +// +// Return Value: +// A type that matches ABIPassingSegment::Size and the register. +// +var_types ABIPassingSegment::GetRegisterType(ClassLayout* layout) const +{ + if (genIsValidIntReg(GetRegister())) + { + assert(Offset < layout->GetSize()); + if (((Offset % TARGET_POINTER_SIZE) == 0) && (Size == TARGET_POINTER_SIZE)) + { + return layout->GetGCPtrType(Offset / TARGET_POINTER_SIZE); + } + } + + return GetRegisterType(); +} + //----------------------------------------------------------------------------- // InRegister: // Create an ABIPassingSegment representing that a segment is passed in a diff --git a/src/coreclr/jit/abi.h b/src/coreclr/jit/abi.h index dcb7b73aacff57..dcb23cfd37470c 100644 --- a/src/coreclr/jit/abi.h +++ b/src/coreclr/jit/abi.h @@ -41,6 +41,7 @@ class ABIPassingSegment unsigned GetStackSize() const; var_types GetRegisterType() const; + var_types GetRegisterType(ClassLayout* layout) const; static ABIPassingSegment InRegister(regNumber reg, unsigned offset, unsigned size); static ABIPassingSegment OnStack(unsigned stackOffset, unsigned offset, unsigned size); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 63baaf1cc386d3..fa0fc56cd759bc 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11633,6 +11633,7 @@ class Compiler #endif // defined(UNIX_AMD64_ABI) bool fgTryMorphStructArg(CallArg* arg); + bool FieldsMatchAbi(LclVarDsc* varDsc, const ABIPassingInformation& abiInfo); bool killGCRefs(GenTree* tree); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index cc33d1c00b0bf7..56be3ec5750fe7 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -2725,16 +2725,10 @@ struct GenTreeFieldList : public GenTree class UseList { - Use* m_head; - Use* m_tail; + Use* m_head = nullptr; + Use* m_tail = nullptr; public: - UseList() - : m_head(nullptr) - , m_tail(nullptr) - { - } - Use* GetHead() const { return m_head; @@ -2792,6 +2786,12 @@ struct GenTreeFieldList : public GenTree } } + void Clear() + { + m_head = nullptr; + m_tail = nullptr; + } + bool IsSorted() const { unsigned offset = 0; diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 5b13b6494351ae..f961a27d8585e3 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1725,20 +1725,22 @@ void Lowering::LowerArg(GenTreeCall* call, CallArg* callArg) { if (abiInfo.HasAnyRegisterSegment()) { -#if FEATURE_MULTIREG_ARGS - if ((abiInfo.NumSegments > 1) && arg->OperIs(GT_FIELD_LIST)) + if (arg->OperIs(GT_FIELD_LIST) || (abiInfo.NumSegments > 1)) { - unsigned int regIndex = 0; - for (GenTreeFieldList::Use& use : arg->AsFieldList()->Uses()) + if (!arg->OperIs(GT_FIELD_LIST)) { - const ABIPassingSegment& segment = abiInfo.Segment(regIndex); - InsertPutArgReg(&use.NodeRef(), segment); - - regIndex++; + // Primitive arg, but the ABI requires it to be split into + // registers. Insert the field list here. + GenTreeFieldList* fieldList = comp->gtNewFieldList(); + fieldList->AddFieldLIR(comp, arg, 0, genActualType(arg->TypeGet())); + BlockRange().InsertAfter(arg, fieldList); + arg = *ppArg = fieldList; } + + LowerArgFieldList(callArg, arg->AsFieldList()); + arg = *ppArg; } else -#endif // FEATURE_MULTIREG_ARGS { assert(abiInfo.HasExactlyOneRegisterSegment()); InsertPutArgReg(ppArg, abiInfo.Segment(0)); @@ -4809,6 +4811,18 @@ void Lowering::LowerRet(GenTreeOp* ret) ContainCheckRet(ret); } +struct LowerFieldListRegisterInfo +{ + unsigned Offset; + var_types RegType; + + LowerFieldListRegisterInfo(unsigned offset, var_types regType) + : Offset(offset) + , RegType(regType) + { + } +}; + //---------------------------------------------------------------------------------------------- // LowerRetFieldList: // Lower a returned FIELD_LIST node. @@ -4822,21 +4836,18 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; unsigned numRegs = retDesc.GetReturnRegCount(); - bool isCompatible = IsFieldListCompatibleWithReturn(fieldList); + auto getRegInfo = [=, &retDesc](unsigned regIndex) { + unsigned offset = retDesc.GetReturnFieldOffset(regIndex); + var_types regType = genActualType(retDesc.GetReturnRegType(regIndex)); + return LowerFieldListRegisterInfo(offset, regType); + }; + + bool isCompatible = IsFieldListCompatibleWithRegisters(fieldList, numRegs, getRegInfo); if (!isCompatible) { - JITDUMP("Spilling field list [%06u] to stack\n", Compiler::dspTreeID(fieldList)); - unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Spilled local for return value")); + unsigned lclNum = + StoreFieldListToNewLocal(comp->typGetObjLayout(comp->info.compMethodInfo->args.retTypeClass), fieldList); LclVarDsc* varDsc = comp->lvaGetDesc(lclNum); - comp->lvaSetStruct(lclNum, comp->info.compMethodInfo->args.retTypeClass, false); - comp->lvaSetVarDoNotEnregister(lclNum DEBUGARG(DoNotEnregisterReason::BlockOpRet)); - - for (GenTreeFieldList::Use& use : fieldList->Uses()) - { - GenTree* store = comp->gtNewStoreLclFldNode(lclNum, use.GetType(), use.GetOffset(), use.GetNode()); - BlockRange().InsertAfter(use.GetNode(), store); - LowerNode(store); - } GenTree* retValue = comp->gtNewLclvNode(lclNum, varDsc->TypeGet()); ret->SetReturnValue(retValue); @@ -4859,7 +4870,89 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) return; } - LowerFieldListToFieldListOfRegisters(fieldList); + LowerFieldListToFieldListOfRegisters(fieldList, numRegs, getRegInfo); +} + +//---------------------------------------------------------------------------------------------- +// StoreFieldListToNewLocal: +// Create a new local with the specified layout and store the specified +// fields of the specified FIELD_LIST into it. +// +// Arguments: +// layout - Layout of the new local +// fieldList - Fields to store to it +// +// Returns: +// Var number of new local. +// +unsigned Lowering::StoreFieldListToNewLocal(ClassLayout* layout, GenTreeFieldList* fieldList) +{ + JITDUMP("Spilling field list [%06u] to stack\n", Compiler::dspTreeID(fieldList)); + unsigned lclNum = comp->lvaGrabTemp(true DEBUGARG("Spilled local for field list")); + LclVarDsc* varDsc = comp->lvaGetDesc(lclNum); + comp->lvaSetStruct(lclNum, layout, false); + comp->lvaSetVarDoNotEnregister(lclNum DEBUGARG(DoNotEnregisterReason::LocalField)); + + for (GenTreeFieldList::Use& use : fieldList->Uses()) + { + GenTree* store = comp->gtNewStoreLclFldNode(lclNum, use.GetType(), use.GetOffset(), use.GetNode()); + BlockRange().InsertAfter(use.GetNode(), store); + LowerNode(store); + } + + return lclNum; +} + +//---------------------------------------------------------------------------------------------- +// LowerArgFieldList: +// Lower an argument FIELD_LIST node. +// +// Arguments: +// arg - The argument +// fieldList - The FIELD_LIST node +// +void Lowering::LowerArgFieldList(CallArg* arg, GenTreeFieldList* fieldList) +{ + assert(!arg->AbiInfo.HasAnyStackSegment()); + + auto getRegInfo = [=](unsigned regIndex) { + const ABIPassingSegment& seg = arg->AbiInfo.Segment(regIndex); + return LowerFieldListRegisterInfo(seg.Offset, seg.GetRegisterType()); + }; + + bool isCompatible = IsFieldListCompatibleWithRegisters(fieldList, arg->AbiInfo.NumSegments, getRegInfo); + if (!isCompatible) + { + ClassLayout* layout = comp->typGetObjLayout(arg->GetSignatureClassHandle()); + unsigned lclNum = StoreFieldListToNewLocal(layout, fieldList); + fieldList->Uses().Clear(); + for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) + { + GenTreeLclFld* fld = comp->gtNewLclFldNode(lclNum, seg.GetRegisterType(layout), seg.Offset); + fieldList->AddFieldLIR(comp, fld, seg.Offset, fld->TypeGet()); + BlockRange().InsertBefore(fieldList, fld); + } + } + else + { + LowerFieldListToFieldListOfRegisters(fieldList, arg->AbiInfo.NumSegments, getRegInfo); + } + + GenTreeFieldList::Use* field = fieldList->Uses().GetHead(); + for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) + { + assert((field != nullptr) && "Ran out of fields while inserting PUTARG_REG"); + InsertPutArgReg(&field->NodeRef(), seg); + field = field->GetNext(); + } + + assert((field == nullptr) && "Missed fields while inserting PUTARG_REG"); + + arg->NodeRef() = fieldList->SoleFieldOrThis(); + if (arg->GetNode() != fieldList) + { + BlockRange().Remove(fieldList); + } } //---------------------------------------------------------------------------------------------- @@ -4874,21 +4967,29 @@ void Lowering::LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList) // True if the fields of the FIELD_LIST are all direct insertions into the // return registers. // -bool Lowering::IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList) +template +bool Lowering::IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, + unsigned numRegs, + GetRegisterInfoFunc getRegInfo) { - JITDUMP("Checking if field list [%06u] is compatible with return ABI: ", Compiler::dspTreeID(fieldList)); - const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; - unsigned numRetRegs = retDesc.GetReturnRegCount(); + JITDUMP("Checking if field list [%06u] is compatible with registers: ", Compiler::dspTreeID(fieldList)); GenTreeFieldList::Use* use = fieldList->Uses().GetHead(); - for (unsigned i = 0; i < numRetRegs; i++) + for (unsigned i = 0; i < numRegs; i++) { - unsigned regStart = retDesc.GetReturnFieldOffset(i); - var_types regType = retDesc.GetReturnRegType(i); - unsigned regEnd = regStart + genTypeSize(regType); + LowerFieldListRegisterInfo regInfo = getRegInfo(i); + unsigned regStart = regInfo.Offset; + var_types regType = regInfo.RegType; + unsigned regEnd = regStart + genTypeSize(regType); + + if ((i == numRegs - 1) && !varTypeUsesFloatReg(regType)) + { + // Allow tail end to pass undefined bits into the register + regEnd = regStart + REGSIZE_BYTES; + } // TODO-CQ: Could just create a 0 for this. - if (use == nullptr) + if ((use == nullptr) || (use->GetOffset() >= regEnd)) { JITDUMP("it is not; register %u has no corresponding field\n", i); return false; @@ -4949,19 +5050,26 @@ bool Lowering::IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList) // Arguments: // fieldList - The field list // -void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList) +template +void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList, + unsigned numRegs, + GetRegisterInfoFunc getRegInfo) { - const ReturnTypeDesc& retDesc = comp->compRetTypeDesc; - unsigned numRegs = retDesc.GetReturnRegCount(); - GenTreeFieldList::Use* use = fieldList->Uses().GetHead(); assert(fieldList->Uses().IsSorted()); for (unsigned i = 0; i < numRegs; i++) { - unsigned regStart = retDesc.GetReturnFieldOffset(i); - var_types regType = genActualType(retDesc.GetReturnRegType(i)); - unsigned regEnd = regStart + genTypeSize(regType); + LowerFieldListRegisterInfo regInfo = getRegInfo(i); + unsigned regStart = regInfo.Offset; + var_types regType = regInfo.RegType; + unsigned regEnd = regStart + genTypeSize(regType); + + if ((i == numRegs - 1) && !varTypeUsesFloatReg(regType)) + { + // Allow tail end to pass undefined bits into the register + regEnd = regStart + REGSIZE_BYTES; + } GenTreeFieldList::Use* regEntry = use; @@ -5001,7 +5109,7 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList) } // If this is a float -> int insertion, then we need the bitcast now. - if (varTypeUsesFloatReg(value) && varTypeUsesIntReg(regType)) + if (varTypeUsesFloatReg(value) && varTypeUsesIntReg(regInfo.RegType)) { assert((genTypeSize(value) == 4) || (genTypeSize(value) == 8)); var_types castType = genTypeSize(value) == 4 ? TYP_INT : TYP_LONG; diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 019a2f2b48a4bd..75507bd6caa1a2 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -187,10 +187,14 @@ class Lowering final : public Phase GenTree* LowerAsyncContinuation(GenTree* asyncCont); void LowerReturnSuspend(GenTree* retSuspend); void LowerRetFieldList(GenTreeOp* ret, GenTreeFieldList* fieldList); - bool IsFieldListCompatibleWithReturn(GenTreeFieldList* fieldList); - void LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList); - void LowerCallStruct(GenTreeCall* call); - void LowerStoreSingleRegCallStruct(GenTreeBlk* store); + unsigned StoreFieldListToNewLocal(ClassLayout* layout, GenTreeFieldList* fieldList); + void LowerArgFieldList(CallArg* arg, GenTreeFieldList* fieldList); + template + bool IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, unsigned numRegs, GetRegisterInfoFunc func); + template + void LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldList, unsigned numRegs, GetRegisterInfoFunc func); + void LowerCallStruct(GenTreeCall* call); + void LowerStoreSingleRegCallStruct(GenTreeBlk* store); #if !defined(WINDOWS_AMD64_ABI) GenTreeLclVar* SpillStructCallResult(GenTreeCall* call) const; #endif // WINDOWS_AMD64_ABI diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index de7fe9bfd19269..7ed36b5f6fc246 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2219,11 +2219,11 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call) // False if the argument cannot be put into a shape supported by the backend. // // Remarks: -// The backend requires register-passed arguments to be of FIELD_LIST shape -// with one primitive node for each register. If the argument is passed in -// only one register, then the FIELD_LIST should be directly replaced by its -// operand. For stack-passed arguments the backend supports struct-typed -// arguments directly. +// The backend requires register-passed arguments to be of FIELD_LIST shape. +// For split arguments it is additionally required that registers and stack +// slots have clean mappings to fields. +// For stack-passed arguments the backend supports struct-typed arguments +// directly. // bool Compiler::fgTryMorphStructArg(CallArg* arg) { @@ -2299,49 +2299,20 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) JITDUMP("Struct argument V%02u: ", lclNum); JITDUMPEXEC(arg->Dump(this)); - // Try to see if we can use the promoted fields to pass this argument. + // Try to see if we can and should use promoted fields to pass this + // argument. // - if (varDsc->lvPromoted && !varDsc->lvDoNotEnregister && - (varDsc->lvFieldCnt == arg->AbiInfo.CountRegsAndStackSlots())) + if (varDsc->lvPromoted && !varDsc->lvDoNotEnregister && (!isSplit || FieldsMatchAbi(varDsc, arg->AbiInfo))) { - bool fieldsMatch = true; - - for (const ABIPassingSegment& seg : arg->AbiInfo.Segments()) - { - if (seg.IsPassedInRegister()) - { - unsigned fieldLclNum = lvaGetFieldLocal(varDsc, seg.Offset); - if (fieldLclNum == BAD_VAR_NUM) - { - fieldsMatch = false; - break; - } - } - else - { - for (unsigned offset = 0; offset < seg.Size; offset += TARGET_POINTER_SIZE) - { - if (lvaGetFieldLocal(varDsc, seg.Offset + offset) == BAD_VAR_NUM) - { - fieldsMatch = false; - break; - } - } - } - } - - if (fieldsMatch) - { - newArg = fgMorphLclToFieldList(lclNode)->SoleFieldOrThis(); - newArg = fgMorphTree(newArg); - } + newArg = fgMorphLclToFieldList(lclNode)->SoleFieldOrThis(); + newArg = fgMorphTree(newArg); } } else if (argNode->OperIsFieldList()) { // We can already see a field list here if physical promotion created it. // Physical promotion will also create single-field field lists which - // the backend does not expect, so fix that here. + // not everything treats the same as a single node, so fix that here. newArg = argNode->AsFieldList()->SoleFieldOrThis(); if (newArg == argNode) { @@ -2353,20 +2324,7 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) // if (newArg == nullptr) { - bool isUseOfIndependentlyPromotedStruct = - argNode->OperIs(GT_LCL_VAR) && - (lvaGetPromotionType(argNode->AsLclVarCommon()->GetLclNum()) == PROMOTION_TYPE_INDEPENDENT); - if (isUseOfIndependentlyPromotedStruct) - { - if (arg->AbiInfo.HasExactlyOneRegisterSegment()) - { - // We already tried to use the fields above, but that failed. - // Here we prefer to create a copy to avoid DNER'ing the local. - // That turns out to be profitable mostly for single-register arguments. - return false; - } - } - else if (!argNode->TypeIs(TYP_STRUCT) && arg->AbiInfo.HasExactlyOneRegisterSegment()) + if (!argNode->TypeIs(TYP_STRUCT) && arg->AbiInfo.HasExactlyOneRegisterSegment()) { // This can be treated primitively. Leave it alone. return true; @@ -2544,6 +2502,51 @@ bool Compiler::fgTryMorphStructArg(CallArg* arg) return true; } +//----------------------------------------------------------------------------- +// FieldsMatchAbi: +// Check if the fields of a local map cleanly (in terms of offsets) to the +// specified ABI info. +// +// Arguments: +// varDsc - promoted local +// abiInfo - ABI information +// +// Returns: +// True if it does. In that case FIELD_LIST usage is allowed for split args +// by the backend. +// +bool Compiler::FieldsMatchAbi(LclVarDsc* varDsc, const ABIPassingInformation& abiInfo) +{ + if (varDsc->lvFieldCnt != abiInfo.CountRegsAndStackSlots()) + { + return false; + } + + for (const ABIPassingSegment& seg : abiInfo.Segments()) + { + if (seg.IsPassedInRegister()) + { + unsigned fieldLclNum = lvaGetFieldLocal(varDsc, seg.Offset); + if (fieldLclNum == BAD_VAR_NUM) + { + return false; + } + } + else + { + for (unsigned offset = 0; offset < seg.Size; offset += TARGET_POINTER_SIZE) + { + if (lvaGetFieldLocal(varDsc, seg.Offset + offset) == BAD_VAR_NUM) + { + return false; + } + } + } + } + + return true; +} + //------------------------------------------------------------------------ // fgMorphLclToFieldList: Morph a GT_LCL_VAR node to a GT_FIELD_LIST of its promoted fields //