diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index c8a142f07b7a69..6f9eba855f8399 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -2904,10 +2904,9 @@ void Compiler::fgLinkBasicBlocks() // fgFirstBB->bbRefs = 1; - // Special args to fgAddRefPred so it will use the initialization fast path. + // Special arg to fgAddRefPred so it will use the initialization fast path. // - FlowEdge* const oldEdge = nullptr; - bool const initializingPreds = true; + const bool initializingPreds = true; for (BasicBlock* const curBBdesc : Blocks()) { @@ -2915,15 +2914,16 @@ void Compiler::fgLinkBasicBlocks() { case BBJ_COND: { - BasicBlock* const trueTarget = fgLookupBB(curBBdesc->GetTargetOffs()); + BasicBlock* const trueTarget = fgLookupBB(curBBdesc->GetTargetOffs()); + BasicBlock* const falseTarget = curBBdesc->Next(); curBBdesc->SetTrueTarget(trueTarget); - curBBdesc->SetFalseTarget(curBBdesc->Next()); - fgAddRefPred(trueTarget, curBBdesc, oldEdge); - fgAddRefPred(curBBdesc->GetFalseTarget(), curBBdesc, oldEdge); + curBBdesc->SetFalseTarget(falseTarget); + fgAddRefPred(trueTarget, curBBdesc); + fgAddRefPred(falseTarget, curBBdesc); - if (curBBdesc->GetTrueTarget()->bbNum <= curBBdesc->bbNum) + if (trueTarget->bbNum <= curBBdesc->bbNum) { - fgMarkBackwardJump(curBBdesc->GetTrueTarget(), curBBdesc); + fgMarkBackwardJump(trueTarget, curBBdesc); } if (curBBdesc->IsLast()) @@ -2945,7 +2945,7 @@ void Compiler::fgLinkBasicBlocks() // Redundantly use SetKindAndTarget() instead of SetTarget() just this once, // so we don't break the HasInitializedTarget() invariant of SetTarget(). curBBdesc->SetKindAndTarget(curBBdesc->GetKind(), jumpDest); - fgAddRefPred(jumpDest, curBBdesc, oldEdge); + fgAddRefPred(jumpDest, curBBdesc); if (curBBdesc->GetTarget()->bbNum <= curBBdesc->bbNum) { @@ -2978,7 +2978,7 @@ void Compiler::fgLinkBasicBlocks() { BasicBlock* jumpDest = fgLookupBB((unsigned)*(size_t*)jumpPtr); *jumpPtr = jumpDest; - fgAddRefPred(jumpDest, curBBdesc, oldEdge); + fgAddRefPred(jumpDest, curBBdesc); if ((*jumpPtr)->bbNum <= curBBdesc->bbNum) { fgMarkBackwardJump(*jumpPtr, curBBdesc); @@ -3839,7 +3839,8 @@ void Compiler::fgFindBasicBlocks() { // Mark catch handler as successor. block->SetTarget(hndBegBB); - fgAddRefPred(hndBegBB, block); + FlowEdge* const newEdge = fgAddRefPred(hndBegBB, block); + newEdge->setLikelihood(1.0); assert(block->GetTarget()->bbCatchTyp == BBCT_FILTER_HANDLER); break; } diff --git a/src/coreclr/jit/fgflow.cpp b/src/coreclr/jit/fgflow.cpp index 1321977e1fd077..e384bc6155890f 100644 --- a/src/coreclr/jit/fgflow.cpp +++ b/src/coreclr/jit/fgflow.cpp @@ -136,6 +136,32 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (flowLast->getSourceBlock() == blockPred) { flow = flowLast; + + // This edge should have been given a likelihood when it was created. + // Since we're increasing its duplicate count, update the likelihood. + // + assert(flow->hasLikelihood()); + const unsigned numSucc = blockPred->NumSucc(); + assert(numSucc > 0); + + if (numSucc == 1) + { + // BasicBlock::NumSucc() returns 1 for BBJ_CONDs with the same true/false target. + // For blocks that only ever have one successor (BBJ_ALWAYS, BBJ_LEAVE, etc.), + // their successor edge should never have a duplicate count over 1. + // + assert(blockPred->KindIs(BBJ_COND)); + assert(blockPred->TrueTargetIs(blockPred->GetFalseTarget())); + flow->setLikelihood(1.0); + } + else + { + // Duplicate count isn't updated until later, so add 1 for now. + // + const unsigned dupCount = flow->getDupCount() + 1; + assert(dupCount > 1); + flow->setLikelihood((1.0 / numSucc) * dupCount); + } } } } @@ -185,12 +211,19 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (initializingPreds) { block->bbLastPred = flow; - } - // Copy likelihood from old edge, if any. - // - if ((oldEdge != nullptr) && oldEdge->hasLikelihood()) + // When initializing preds, ensure edge likelihood is set, + // such that this edge is as likely as any other successor edge + // + const unsigned numSucc = blockPred->NumSucc(); + assert(numSucc > 0); + assert(flow->getDupCount() == 1); + flow->setLikelihood(1.0 / numSucc); + } + else if ((oldEdge != nullptr) && oldEdge->hasLikelihood()) { + // Copy likelihood from old edge, if any. + // flow->setLikelihood(oldEdge->getLikelihood()); } @@ -235,6 +268,10 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE // assert(block->checkPredListOrder()); + // When initializing preds, edge likelihood should always be set. + // + assert(!initializingPreds || flow->hasLikelihood()); + return flow; } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 3e636b4d77c191..9e29cc25792734 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -3916,7 +3916,10 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, FlowEdge* const flowEdge = m_comp->fgGetPredForBlock(edge->m_targetBlock, block); assert(flowEdge != nullptr); - assert(!flowEdge->hasLikelihood()); + + // Naive likelihood should have been set during pred initialization in fgAddRefPred + // + assert(flowEdge->hasLikelihood()); weight_t likelihood = 0; if (nEdges == 1) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 3ba9e845fc8abc..86d04c064334b7 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -12280,7 +12280,7 @@ void Compiler::impFixPredLists() BasicBlock* const continuation = predBlock->Next(); FlowEdge* const newEdge = fgAddRefPred(continuation, finallyBlock); - newEdge->setLikelihood(1.0); + newEdge->setLikelihood(1.0 / predCount); jumpEhf->bbeSuccs[predNum] = continuation; ++predNum;