diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 7b3b895f54c41e..d8e501d4b10514 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -76,6 +76,9 @@ PhaseStatus Compiler::SaveAsyncContexts() lvaAsyncSynchronizationContextVar = lvaGrabTemp(false DEBUGARG("Async SynchronizationContext")); lvaGetDesc(lvaAsyncSynchronizationContextVar)->lvType = TYP_REF; + lvaResumedIndicator = lvaGrabTemp(false DEBUGARG("Async Resumed")); + lvaGetDesc(lvaResumedIndicator)->lvType = TYP_UBYTE; + if (opts.IsOSR()) { lvaGetDesc(lvaAsyncThreadObjectVar)->lvIsOSRLocal = true; @@ -165,7 +168,17 @@ PhaseStatus Compiler::SaveAsyncContexts() // Insert CaptureContexts call before the try (keep it before so the // try/finally can be removed if there is no exception side effects). // For OSR, we did this in the tier0 method. - if (!opts.IsOSR()) + if (opts.IsOSR()) + { + // In the OSR method we compute the initial value of the resumption indicator based on the continuation arg. + GenTree* continuation = gtNewLclVarNode(lvaAsyncContinuationArg, TYP_REF); + GenTree* null = gtNewNull(); + GenTree* contNeNull = gtNewOperNode(GT_NE, TYP_INT, continuation, null); + GenTree* storeIndicator = gtNewStoreLclVarNode(lvaResumedIndicator, contNeNull); + Statement* storeIndicatorStmt = fgNewStmtFromTree(storeIndicator); + fgInsertStmtAtBeg(fgFirstBB, storeIndicatorStmt); + } + else { GenTreeCall* captureCall = gtNewUserCallNode(asyncInfo->captureContextsMethHnd, TYP_VOID); captureCall->gtArgs.PushFront(this, @@ -186,21 +199,30 @@ PhaseStatus Compiler::SaveAsyncContexts() JITDUMP("Inserted capture\n"); DISPSTMT(captureStmt); + + // Also initialize resumed indicator var if it will not be initialized by the prolog. + BasicBlock* containingBlock = compIsForInlining() ? impInlineInfo->iciBlock : fgFirstBB; + bool inALoop = containingBlock->HasFlag(BBF_BACKWARD_JUMP); + bool isReturn = containingBlock->KindIs(BBJ_RETURN); + + if ((inALoop && !isReturn) || !impInlineRoot()->info.compInitMem) + { + GenTree* storeIndicator = gtNewStoreLclVarNode(lvaResumedIndicator, gtNewIconNode(0)); + Statement* storeIndicatorStmt = fgNewStmtFromTree(storeIndicator); + fgInsertStmtAtBeg(fgFirstBB, storeIndicatorStmt); + + JITDUMP("Inserted resumed indicator initialization\n"); + DISPSTMT(storeIndicatorStmt); + } + else + { + JITDUMP("Skipping zero init of resumed indicator due to compInitMem\n"); + } } // Insert RestoreContexts call in fault (exceptional case) // First argument: resumed = (continuation != null) - GenTree* resumed; - if (compIsForInlining()) - { - resumed = gtNewFalse(); - } - else - { - GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); - GenTree* null = gtNewNull(); - resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); - } + GenTree* resumed = gtNewLclvNode(lvaResumedIndicator, TYP_INT); GenTreeCall* restoreCall = gtNewUserCallNode(asyncInfo->restoreContextsMethHnd, TYP_VOID); restoreCall->gtArgs.PushFront(this, @@ -315,15 +337,23 @@ void Compiler::AddContextArgsToAsyncCalls(BasicBlock* block) return WALK_CONTINUE; } - GenTreeCall* call = tree->AsCall(); - GenTree* execCtx = m_compiler->gtNewLclVarNode(m_compiler->lvaAsyncExecutionContextVar, TYP_REF); + GenTreeCall* call = tree->AsCall(); + GenTree* resumed = m_compiler->gtNewLclVarNode(m_compiler->lvaResumedIndicator, TYP_INT); + GenTree* resumedAddr = m_compiler->gtNewLclAddrNode(m_compiler->lvaResumedIndicator, 0); + GenTree* execCtx = m_compiler->gtNewLclVarNode(m_compiler->lvaAsyncExecutionContextVar, TYP_REF); GenTree* syncCtx = m_compiler->gtNewLclVarNode(m_compiler->lvaAsyncSynchronizationContextVar, TYP_REF); - JITDUMP("Adding exec context [%06u], sync context [%06u] to async call [%06u]\n", dspTreeID(execCtx), - dspTreeID(syncCtx), dspTreeID(call)); + JITDUMP( + "Adding resumed use [%06u], resumed def [%06u] exec context [%06u], sync context [%06u] to async call [%06u]\n", + dspTreeID(resumed), dspTreeID(resumedAddr), dspTreeID(execCtx), dspTreeID(syncCtx), dspTreeID(call)); call->gtArgs.PushFront(m_compiler, NewCallArg::Primitive(syncCtx).WellKnown(WellKnownArg::AsyncSynchronizationContext)); call->gtArgs.PushFront(m_compiler, NewCallArg::Primitive(execCtx).WellKnown(WellKnownArg::AsyncExecutionContext)); + call->gtArgs.PushFront(m_compiler, + NewCallArg::Primitive(resumedAddr).WellKnown(WellKnownArg::AsyncResumedDef)); + call->gtArgs.PushFront(m_compiler, NewCallArg::Primitive(resumed).WellKnown(WellKnownArg::AsyncResumedUse)); + + m_compiler->lvaGetDesc(m_compiler->lvaResumedIndicator)->lvHasLdAddrOp = true; return WALK_CONTINUE; } }; @@ -357,17 +387,7 @@ BasicBlock* Compiler::CreateReturnBB(unsigned* mergedReturnLcl) // Insert "restore" call CORINFO_ASYNC_INFO* asyncInfo = eeGetAsyncInfo(); - GenTree* resumed; - if (compIsForInlining()) - { - resumed = gtNewFalse(); - } - else - { - GenTree* continuation = gtNewLclvNode(lvaAsyncContinuationArg, TYP_REF); - GenTree* null = gtNewNull(); - resumed = gtNewOperNode(GT_NE, TYP_INT, continuation, null); - } + GenTree* resumed = gtNewLclvNode(lvaResumedIndicator, TYP_INT); GenTreeCall* restoreCall = gtNewUserCallNode(asyncInfo->restoreContextsMethHnd, TYP_VOID); restoreCall->gtArgs.PushFront(this, @@ -744,11 +764,13 @@ PhaseStatus AsyncTransformation::Run() } } + GenTreeLclVarCommon* commonAsyncResumedDef = FindAndRemoveCommonAsyncResumedDef(); + CreateResumptionsAndSuspensions(); // After transforming all async calls we have created resumption blocks; // create the resumption switch. - CreateResumptionSwitch(); + CreateResumptionSwitch(commonAsyncResumedDef); m_compiler->fgInvalidateDfsTree(); @@ -1075,6 +1097,10 @@ void AsyncTransformation::CreateLiveSetForSuspension(BasicBlock* { excludedLocals.AddOrUpdate(m_compiler->lvaAsyncExecutionContextVar, true); } + if (m_compiler->lvaResumedIndicator != BAD_VAR_NUM) + { + excludedLocals.AddOrUpdate(m_compiler->lvaResumedIndicator, true); + } analyses.GetLiveLocals(layoutBuilder, [&](unsigned lclNum) { return !excludedLocals.Contains(lclNum); @@ -1920,7 +1946,8 @@ bool AsyncTransformation::IsReusableSuspension(const AsyncState* state, } } - static const WellKnownArg validateArgs[] = {WellKnownArg::AsyncExecutionContext, + static const WellKnownArg validateArgs[] = {WellKnownArg::AsyncResumedUse, WellKnownArg::AsyncResumedDef, + WellKnownArg::AsyncExecutionContext, WellKnownArg::AsyncSynchronizationContext}; for (WellKnownArg arg : validateArgs) { @@ -1935,7 +1962,10 @@ bool AsyncTransformation::IsReusableSuspension(const AsyncState* state, if (thisArg != nullptr) { - if (!thisArg->GetNode()->OperIsLocal()) + // The value may have been folded to a constant (e.g. when the + // indicator is provably zero at this point), which is still fine to + // compare and to remove from the call. + if (!thisArg->GetNode()->OperIsAnyLocal() && !thisArg->GetNode()->IsInvariant()) { JITDUMP(" No; %s argument is too complex\n", getWellKnownArgName(arg)); return false; @@ -1972,14 +2002,15 @@ bool AsyncTransformation::IsReusableSuspension(const AsyncState* state, // void AsyncTransformation::HandleReusedSuspension(BasicBlock* callBlock, GenTreeCall* call) { - static const WellKnownArg argsToRemove[] = {WellKnownArg::AsyncExecutionContext, + static const WellKnownArg argsToRemove[] = {WellKnownArg::AsyncResumedUse, WellKnownArg::AsyncResumedDef, + WellKnownArg::AsyncExecutionContext, WellKnownArg::AsyncSynchronizationContext}; for (WellKnownArg wka : argsToRemove) { CallArg* arg = call->gtArgs.FindWellKnownArg(wka); if (arg != nullptr) { - assert(arg->GetNode()->OperIsLocal()); + assert(arg->GetNode()->OperIsAnyLocal() || arg->GetNode()->IsInvariant()); LIR::AsRange(callBlock).Remove(arg->GetNode()); call->gtArgs.RemoveUnsafe(arg); } @@ -2563,15 +2594,27 @@ void AsyncTransformation::FinishContextHandlingAndSuspensionWithHelper(BasicBloc ? m_sharedFinishContextHandlingWithContinuationContextBB : m_sharedFinishContextHandlingWithoutContinuationContextBB; + CallArg* resumedArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedUse); CallArg* execContextArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncExecutionContext); CallArg* syncContextArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncSynchronizationContext); - assert((execContextArg != nullptr) && (syncContextArg != nullptr)); + assert((resumedArg != nullptr) && (execContextArg != nullptr) && (syncContextArg != nullptr)); // Get the contexts from the call node: // 1. For shared finish, store it directly to the shared locals in the same block // 2. For non-shared finish, just make sure it is a GT_LCL_VAR since we need to create // a use in a different block. // Also remove the nodes from the original block and the call args. + GenTree* resumed = resumedArg->GetNode(); + if (!resumed->IsInvariant() && !resumed->OperIs(GT_LCL_VAR)) + { + // We are moving resumed into a different BB so create a temp for it. + LIR::Use use(LIR::AsRange(callBlock), &resumedArg->NodeRef(), call); + use.ReplaceWithLclVar(m_compiler); + resumed = use.Def(); + } + LIR::AsRange(callBlock).Remove(resumed); + call->gtArgs.RemoveUnsafe(resumedArg); + GenTree* execContext = execContextArg->GetNode(); if (!execContext->OperIs(GT_LCL_VAR)) { @@ -2596,7 +2639,13 @@ void AsyncTransformation::FinishContextHandlingAndSuspensionWithHelper(BasicBloc if (sharedFinish != nullptr) { - // Store the contexts to the shared locals that the shared finish block will take them from. + // Store the vars to the shared locals that the shared finish block will take them from. + if (m_sharedFinishContextHandlingResumedVar != BAD_VAR_NUM) + { + GenTree* storeResumed = m_compiler->gtNewStoreLclVarNode(m_sharedFinishContextHandlingResumedVar, resumed); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, storeResumed)); + } + if (m_sharedFinishContextHandlingExecContextVar != BAD_VAR_NUM) { GenTree* storeExecContext = @@ -2617,7 +2666,7 @@ void AsyncTransformation::FinishContextHandlingAndSuspensionWithHelper(BasicBloc else { // Otherwise insert a new call - InsertFinishContextHandlingCall(suspendBB, layout, helper, execContext, syncContext); + InsertFinishContextHandlingCall(suspendBB, layout, helper, resumed, execContext, syncContext); // And return either via a new GT_RETURN_SUSPEND or via the shared return BB. if (m_sharedReturnBB != nullptr) @@ -2644,10 +2693,12 @@ void AsyncTransformation::FinishContextHandlingAndSuspensionWithHelper(BasicBloc // void AsyncTransformation::RestoreContexts(BasicBlock* block, GenTreeCall* call, BasicBlock* suspendBB) { + CallArg* resumedArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedUse); CallArg* execContextArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncExecutionContext); CallArg* syncContextArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncSynchronizationContext); + assert((resumedArg != nullptr) == (execContextArg != nullptr)); assert((execContextArg != nullptr) == (syncContextArg != nullptr)); - if (execContextArg == nullptr) + if (resumedArg == nullptr) { JITDUMP(" Call [%06u] does not have async contexts; skipping restore on suspension\n", Compiler::dspTreeID(call)); @@ -2673,19 +2724,27 @@ void AsyncTransformation::RestoreContexts(BasicBlock* block, GenTreeCall* call, LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_compiler, restoreCall)); - // Replace resumedPlaceholder with actual "continuationParameter != null" arg + // Replace resumedPlaceholder with actual resumed arg + GenTree* resumed = resumedArg->GetNode(); + if (!resumed->IsInvariant() && !resumed->OperIs(GT_LCL_VAR)) + { + // We are moving resumed into a different BB so create a temp for it. + LIR::Use use(LIR::AsRange(block), &resumedArg->NodeRef(), call); + use.ReplaceWithLclVar(m_compiler); + resumed = use.Def(); + } + LIR::Use use; bool gotUse = LIR::AsRange(suspendBB).TryGetUse(resumedPlaceholder, &use); assert(gotUse); - GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - GenTree* null = m_compiler->gtNewNull(); - GenTree* resumed = m_compiler->gtNewOperNode(GT_NE, TYP_INT, continuation, null); - - LIR::AsRange(suspendBB).InsertBefore(resumedPlaceholder, LIR::SeqTree(m_compiler, resumed)); + LIR::AsRange(block).Remove(resumed); + LIR::AsRange(suspendBB).InsertBefore(resumedPlaceholder, resumed); use.ReplaceWith(resumed); LIR::AsRange(suspendBB).Remove(resumedPlaceholder); + call->gtArgs.RemoveUnsafe(resumedArg); + // Replace execContextPlaceholder with actual value GenTree* execContext = execContextArg->GetNode(); if (!execContext->OperIs(GT_LCL_VAR)) @@ -2883,6 +2942,8 @@ void AsyncTransformation::CreateResumption(BasicBlock* call RestoreFromDataOnResumption(layout, subLayout, resumeBB); } + StoreResumedDef(callBlock, call, resumeBB); + BasicBlock* storeResultBB = resumeBB; if (subLayout.NeedsException()) @@ -2961,6 +3022,70 @@ void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& } } +//------------------------------------------------------------------------ +// AsyncTransformation::StoreResumedDef: +// Assign the resumed def to 1 from the resumption path. +// +// Parameters: +// callBlock - The basic block containing the async call +// call - The async call node +// resumeBB - The basic block to append IR to +// +void AsyncTransformation::StoreResumedDef(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* resumeBB) +{ + CallArg* resumedDefArg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedDef); + + if (resumedDefArg == nullptr) + { + return; + } + + GenTreeLclVarCommon* resumedDef = m_compiler->gtCallGetDefinedAsyncResumedLclAddr(call); + assert((resumedDef != nullptr) && (resumedDefArg->GetNode() == resumedDef)); + + StoreResumedDef(resumedDef, resumeBB); + + LIR::AsRange(callBlock).Remove(resumedDef); + call->gtArgs.RemoveUnsafe(resumedDefArg); +} + +//------------------------------------------------------------------------ +// AsyncTransformation::StoreResumedDef: +// Assign the resumed def to 1 in the specified block. +// +// Parameters: +// resumedDef - The local variable representing the resumed def +// block - The basic block to append IR to +// +void AsyncTransformation::StoreResumedDef(GenTreeLclVarCommon* resumedDef, BasicBlock* block) +{ + JITDUMP(" Have resume def [%06u] to store to\n", Compiler::dspTreeID(resumedDef)); + + LclVarDsc* varDsc = m_compiler->lvaGetDesc(resumedDef); + GenTree* store; + if ((resumedDef->GetLclOffs() == 0) && varDsc->TypeIs(TYP_UBYTE)) + { + store = m_compiler->gtNewStoreLclVarNode(resumedDef->GetLclNum(), m_compiler->gtNewIconNode(1)); + } + else + { + store = m_compiler->gtNewStoreLclFldNode(resumedDef->GetLclNum(), TYP_UBYTE, resumedDef->GetLclOffs(), + m_compiler->gtNewIconNode(1)); + m_compiler->lvaSetVarDoNotEnregister(resumedDef->GetLclNum() DEBUGARG(DoNotEnregisterReason::LocalField)); + } + + if (block->HasTerminator()) + { + LIR::AsRange(block).InsertBefore(block->lastNode(), LIR::SeqTree(m_compiler, store)); + } + else + { + LIR::AsRange(block).InsertAtEnd(LIR::SeqTree(m_compiler, store)); + } + + JITDUMP(" Created store [%06u] to set resumed def to 1\n", Compiler::dspTreeID(store)); +} + //------------------------------------------------------------------------ // AsyncTransformation::RethrowExceptionOnResumption: // Create IR that checks for an exception and rethrows it at the original @@ -3452,6 +3577,7 @@ void AsyncTransformation::CreateSharedReturnBB() // Parameters: // helper - The type of helper to call // layout - The continuation layout +// invariantResumed - Tree node to clone and use for "resumed" computation // execContextMayVary - If true, callers may use different execution // contexts, and thus we need a local to allow it to vary. // syncContextMayVary - If true, callers may use different synchronization @@ -3462,6 +3588,7 @@ void AsyncTransformation::CreateSharedReturnBB() // BasicBlock* AsyncTransformation::CreateSharedFinishContextHandlingBB(SuspensionContextHelper helper, const ContinuationLayout& layout, + GenTree* invariantResumed, bool execContextMayVary, bool syncContextMayVary) { @@ -3479,6 +3606,23 @@ BasicBlock* AsyncTransformation::CreateSharedFinishContextHandlingBB(SuspensionC block->SetFlags(BBF_PROF_WEIGHT); } + GenTree* resumed; + if (invariantResumed == nullptr) + { + if (m_sharedFinishContextHandlingResumedVar == BAD_VAR_NUM) + { + m_sharedFinishContextHandlingResumedVar = + m_compiler->lvaGrabTemp(false DEBUGARG("'resumed' for shared finish context handling")); + m_compiler->lvaGetDesc(m_sharedFinishContextHandlingResumedVar)->lvType = TYP_UBYTE; + } + + resumed = m_compiler->gtNewLclVarNode(m_sharedFinishContextHandlingResumedVar, TYP_UBYTE); + } + else + { + resumed = m_compiler->gtCloneExpr(invariantResumed); + } + unsigned execContextLclNum; if (execContextMayVary) { @@ -3513,7 +3657,8 @@ BasicBlock* AsyncTransformation::CreateSharedFinishContextHandlingBB(SuspensionC syncContextLclNum = m_compiler->lvaAsyncSynchronizationContextVar; } - InsertFinishContextHandlingCall(block, layout, helper, m_compiler->gtNewLclvNode(execContextLclNum, TYP_REF), + InsertFinishContextHandlingCall(block, layout, helper, resumed, + m_compiler->gtNewLclvNode(execContextLclNum, TYP_REF), m_compiler->gtNewLclvNode(syncContextLclNum, TYP_REF)); return block; @@ -3527,12 +3672,14 @@ BasicBlock* AsyncTransformation::CreateSharedFinishContextHandlingBB(SuspensionC // block - Block that should contain the call (inserted at the end) // layout - The continuation layout // helper - The type of helper +// resumed - The resumed tree to pass to the helper // execContext - The execution context tree to pass to the helper // syncContext - The synchronization context tree to pass to the helper // void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* block, const ContinuationLayout& layout, SuspensionContextHelper helper, + GenTree* resumed, GenTree* execContext, GenTree* syncContext) { @@ -3624,15 +3771,11 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* use.ReplaceWith(execContextAddrOffset); LIR::AsRange(block).Remove(execContextAddrPlaceholder); - // Replace resumedPlaceholder with actual "continuationParameter != null" arg + // Replace resumedPlaceholder with the resumed value gotUse = LIR::AsRange(block).TryGetUse(resumedPlaceholder, &use); assert(gotUse); - GenTree* continuation = m_compiler->gtNewLclvNode(m_compiler->lvaAsyncContinuationArg, TYP_REF); - GenTree* null = m_compiler->gtNewNull(); - GenTree* resumed = m_compiler->gtNewOperNode(GT_NE, TYP_INT, continuation, null); - - LIR::AsRange(block).InsertBefore(resumedPlaceholder, LIR::SeqTree(m_compiler, resumed)); + LIR::AsRange(block).InsertBefore(resumedPlaceholder, resumed); use.ReplaceWith(resumed); LIR::AsRange(block).Remove(resumedPlaceholder); @@ -3656,6 +3799,52 @@ void AsyncTransformation::InsertFinishContextHandlingCall(BasicBlock* DISPTREERANGE(LIR::AsRange(block), finishCall); } +//------------------------------------------------------------------------ +// AsyncTransformation::FindAndRemoveCommonAsyncResumedDef: +// If all async calls define the same async resumption indicator variable, +// then remove the def from all calls and return it. +// +// Returns: +// The common def, or null if there is no common def. +// +GenTreeLclVarCommon* AsyncTransformation::FindAndRemoveCommonAsyncResumedDef() +{ + if (m_states.size() <= 1) + { + return nullptr; + } + + GenTreeLclVarCommon* commonDef = nullptr; + + for (const AsyncState& state : m_states) + { + GenTreeLclVarCommon* def = m_compiler->gtCallGetDefinedAsyncResumedLclAddr(state.Call); + if (def == nullptr) + { + return nullptr; + } + + if ((commonDef != nullptr) && !GenTree::Compare(def, commonDef)) + { + return nullptr; + } + + commonDef = def; + } + + JITDUMP(" Found common async resumed def node:\n"); + DISPTREE(commonDef); + + for (const AsyncState& state : m_states) + { + CallArg* arg = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedDef); + LIR::AsRange(state.CallBlock).Remove(arg->GetNode()); + state.Call->gtArgs.RemoveUnsafe(arg); + } + + return commonDef; +} + //------------------------------------------------------------------------ // AsyncTransformation::CreateResumptionsAndSuspensions: // Walk all recorded async states and create the suspension and resumption @@ -3676,8 +3865,10 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() unsigned numSharedSuspensionsWithContinuationContext = 0; unsigned numSharedSuspensionsWithoutContinuationContext = 0; - bool execContextMayVary = false; - bool syncContextMayVary = false; + bool resumedMayVary = false; + bool execContextMayVary = false; + bool syncContextMayVary = false; + GenTree* invariantResumed = nullptr; for (const AsyncState& state : m_states) { @@ -3699,6 +3890,26 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() // unnecessary additional register moves. This is a common case. if (helper != SuspensionContextHelper::None) { + CallArg* resumedArg = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedUse); + assert(resumedArg != nullptr); + GenTree* resumed = resumedArg->GetNode(); + + if (resumed->IsInvariant() || resumed->OperIs(GT_LCL_VAR)) + { + if ((invariantResumed == nullptr) || GenTree::Compare(invariantResumed, resumed)) + { + invariantResumed = resumed; + } + else + { + resumedMayVary = true; + } + } + else + { + resumedMayVary = true; + } + CallArg* execContextArg = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncExecutionContext); CallArg* syncContextArg = state.Call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncSynchronizationContext); @@ -3719,7 +3930,8 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() numSharedSuspensionsWithContinuationContext); m_sharedFinishContextHandlingWithContinuationContextBB = CreateSharedFinishContextHandlingBB(SuspensionContextHelper::WithContinuationContext, *sharedLayout, - execContextMayVary, syncContextMayVary); + resumedMayVary ? nullptr : invariantResumed, execContextMayVary, + syncContextMayVary); } if (numSharedSuspensionsWithoutContinuationContext > 1) @@ -3729,7 +3941,8 @@ void AsyncTransformation::CreateResumptionsAndSuspensions() numSharedSuspensionsWithoutContinuationContext); m_sharedFinishContextHandlingWithoutContinuationContextBB = CreateSharedFinishContextHandlingBB(SuspensionContextHelper::WithoutContinuationContext, *sharedLayout, - execContextMayVary, syncContextMayVary); + resumedMayVary ? nullptr : invariantResumed, execContextMayVary, + syncContextMayVary); } } @@ -3834,7 +4047,7 @@ ContinuationLayoutBuilder* ContinuationLayoutBuilder::CreateSharedLayout(Compile // Create the IR for the entry of the function that checks the continuation // and dispatches on its state number. // -void AsyncTransformation::CreateResumptionSwitch() +void AsyncTransformation::CreateResumptionSwitch(GenTreeLclVarCommon* commonAsyncResumedDef) { m_compiler->fgCreateNewInitBB(); BasicBlock* newEntryBB = m_compiler->fgFirstBB; @@ -3932,6 +4145,14 @@ void AsyncTransformation::CreateResumptionSwitch() resumingEdge->setLikelihood(0); newEntryBB->GetFalseEdge()->setLikelihood(1); + if (commonAsyncResumedDef != nullptr) + { + // If we have a common async resumption def (common), then we do a + // manual head merge to move it into the switch block to avoid storing + // it in every resumption. + StoreResumedDef(commonAsyncResumedDef, resumingEdge->getDestinationBlock()); + } + if (m_compiler->doesMethodHavePatchpoints()) { JITDUMP(" Method has patch points...\n"); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index fb9abf2d8be779..ea9f3943b1b3bb 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -368,7 +368,8 @@ class AsyncTransformation // saves/restores and then suspend. BasicBlock* m_sharedFinishContextHandlingWithContinuationContextBB = nullptr; BasicBlock* m_sharedFinishContextHandlingWithoutContinuationContextBB = nullptr; - // Variables that shared suspension finishing BBs take the exec/sync contexts in + // Variables that shared suspension finishing BBs take the resumed/exec/sync contexts in + unsigned m_sharedFinishContextHandlingResumedVar = BAD_VAR_NUM; unsigned m_sharedFinishContextHandlingExecContextVar = BAD_VAR_NUM; unsigned m_sharedFinishContextHandlingSyncContextVar = BAD_VAR_NUM; @@ -471,6 +472,8 @@ class AsyncTransformation void RestoreFromDataOnResumption(const ContinuationLayout& layout, const ContinuationLayoutBuilder& subLayout, BasicBlock* resumeBB); + void StoreResumedDef(BasicBlock* callBlock, GenTreeCall* call, BasicBlock* resumeBB); + void StoreResumedDef(GenTreeLclVarCommon* resumedDef, BasicBlock* block); BasicBlock* RethrowExceptionOnResumption(BasicBlock* block, const ContinuationLayout& layout, BasicBlock* resumeBB); void CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, @@ -497,16 +500,20 @@ class AsyncTransformation void CreateSharedReturnBB(); BasicBlock* CreateSharedFinishContextHandlingBB(SuspensionContextHelper helper, const ContinuationLayout& layout, + GenTree* invariantResumed, bool execContextMayVary, bool syncContextMayVary); void InsertFinishContextHandlingCall(BasicBlock* block, const ContinuationLayout& layout, SuspensionContextHelper helper, + GenTree* resumed, GenTree* execContext, GenTree* syncContext); bool ReuseContinuations(); - void CreateResumptionsAndSuspensions(); - void CreateResumptionSwitch(); + + GenTreeLclVarCommon* FindAndRemoveCommonAsyncResumedDef(); + void CreateResumptionsAndSuspensions(); + void CreateResumptionSwitch(GenTreeLclVarCommon* commonAsyncResumedDef); public: AsyncTransformation(Compiler* comp) diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 38e96f3ba8f9cc..0e5dae4e7ac605 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -10383,7 +10383,8 @@ bool Compiler::lvaIsOSRLocal(unsigned varNum) // Sanity check for promoted fields of OSR locals. // if ((varNum >= info.compLocalsCount) && (varNum != lvaMonAcquired) && (varNum != lvaAsyncThreadObjectVar) && - (varNum != lvaAsyncExecutionContextVar) && (varNum != lvaAsyncSynchronizationContextVar)) + (varNum != lvaResumedIndicator) && (varNum != lvaAsyncExecutionContextVar) && + (varNum != lvaAsyncSynchronizationContextVar)) { assert(varDsc->lvIsStructField); assert(varDsc->lvParentLcl < info.compLocalsCount); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index adadf7f6a55ca5..e02df0b5f4d930 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4108,6 +4108,7 @@ class Compiler bool gtIsTypeof(GenTree* tree, CORINFO_CLASS_HANDLE* handle = nullptr); GenTreeLclVarCommon* gtCallGetDefinedRetBufLclAddr(GenTreeCall* call); + GenTreeLclVarCommon* gtCallGetDefinedAsyncResumedLclAddr(GenTreeCall* call); //------------------------------------------------------------------------- // Functions to display the trees @@ -4386,6 +4387,9 @@ class Compiler // Variable representing async continuation argument passed. unsigned lvaAsyncContinuationArg = BAD_VAR_NUM; + // Variable representing "have we resumed?" for async methods + unsigned lvaResumedIndicator = BAD_VAR_NUM; + #if defined(DEBUG) && defined(TARGET_XARCH) unsigned lvaReturnSpCheck = BAD_VAR_NUM; // Stores SP to confirm it is not corrupted on return. diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index ecd0a3eeb94024..34fdb90ac0aeb6 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -4618,15 +4618,26 @@ GenTree::VisitResult GenTree::VisitLocalDefs(Compiler* comp, TVisitor visitor) } if (OperIs(GT_CALL)) { - GenTreeCall* call = AsCall(); - GenTreeLclVarCommon* lclAddr = comp->gtCallGetDefinedRetBufLclAddr(call); - if (lclAddr != nullptr) + GenTreeCall* call = AsCall(); + + GenTreeLclVarCommon* asyncResumedLclAddr = comp->gtCallGetDefinedAsyncResumedLclAddr(call); + if (asyncResumedLclAddr != nullptr) + { + bool isEntire = comp->lvaLclExactSize(asyncResumedLclAddr->GetLclNum()) == 1; + + RETURN_IF_ABORT( + visitor(LocalDef(asyncResumedLclAddr, isEntire, asyncResumedLclAddr->GetLclOffs(), ValueSize(1)))); + } + + GenTreeLclVarCommon* retBufLclAddr = comp->gtCallGetDefinedRetBufLclAddr(call); + if (retBufLclAddr != nullptr) { unsigned storeSize = comp->typGetObjLayout(AsCall()->gtRetClsHnd)->GetSize(); - bool isEntire = comp->IsEntireAccess(lclAddr->GetLclNum(), lclAddr->GetLclOffs(), ValueSize(storeSize)); + bool isEntire = + comp->IsEntireAccess(retBufLclAddr->GetLclNum(), retBufLclAddr->GetLclOffs(), ValueSize(storeSize)); - return visitor(LocalDef(lclAddr, isEntire, lclAddr->GetLclOffs(), ValueSize(storeSize))); + return visitor(LocalDef(retBufLclAddr, isEntire, retBufLclAddr->GetLclOffs(), ValueSize(storeSize))); } } @@ -4661,11 +4672,18 @@ GenTree::VisitResult GenTree::VisitLocalDefNodes(Compiler* comp, TVisitor visito } if (OperIs(GT_CALL)) { - GenTreeCall* call = AsCall(); - GenTreeLclVarCommon* lclAddr = comp->gtCallGetDefinedRetBufLclAddr(call); - if (lclAddr != nullptr) + GenTreeCall* call = AsCall(); + + GenTreeLclVarCommon* asyncResumedLclAddr = comp->gtCallGetDefinedAsyncResumedLclAddr(call); + if (asyncResumedLclAddr != nullptr) + { + RETURN_IF_ABORT(visitor(asyncResumedLclAddr)); + } + + GenTreeLclVarCommon* retBufLclAddr = comp->gtCallGetDefinedRetBufLclAddr(call); + if (retBufLclAddr != nullptr) { - return visitor(lclAddr); + return visitor(retBufLclAddr); } } diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index f2d19748e9545a..3f8cb92a7dc19b 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -3838,11 +3838,7 @@ void Compiler::fgDebugCheckLinkedLocals() GenTree* node = *use; if (ShouldLink(node)) { - if ((user != nullptr) && user->IsCall() && - (node == m_compiler->gtCallGetDefinedRetBufLclAddr(user->AsCall()))) - { - } - else + if ((user == nullptr) || !user->IsCall() || !IsDefinedByCall(user->AsCall(), node)) { m_locals.Push(node); } @@ -3850,16 +3846,25 @@ void Compiler::fgDebugCheckLinkedLocals() if (node->IsCall()) { - GenTree* defined = m_compiler->gtCallGetDefinedRetBufLclAddr(node->AsCall()); - if (defined != nullptr) - { - assert(ShouldLink(defined)); - m_locals.Push(defined); - } + auto linkDefs = [&](GenTree* def) { + assert(ShouldLink(def)); + m_locals.Push(def); + return GenTree::VisitResult::Continue; + }; + + node->VisitLocalDefNodes(m_compiler, linkDefs); } return WALK_CONTINUE; } + + bool IsDefinedByCall(GenTreeCall* call, GenTree* node) + { + auto defIsNode = [=](GenTree* def) { + return node == def ? GenTree::VisitResult::Abort : GenTree::VisitResult::Continue; + }; + return call->VisitLocalDefNodes(m_compiler, defIsNode) == GenTree::VisitResult::Abort; + } }; DebugLocalSequencer seq(this); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index a1357523a02804..c14dff145045ec 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1650,6 +1650,8 @@ bool CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension cc, We case WellKnownArg::StackArrayLocal: case WellKnownArg::AsyncExecutionContext: case WellKnownArg::AsyncSynchronizationContext: + case WellKnownArg::AsyncResumedUse: + case WellKnownArg::AsyncResumedDef: // These are pseudo-args; they are not actual arguments, but we // reuse the argument mechanism to represent them as arbitrary uses // that are later expanded out. @@ -21631,6 +21633,34 @@ GenTreeLclVarCommon* Compiler::gtCallGetDefinedRetBufLclAddr(GenTreeCall* call) return node->AsLclVarCommon(); } +//------------------------------------------------------------------------ +// gtCallGetDefinedAsyncResumedLclAddr: +// Get the tree corresponding to the address of the async resumed indicator that this call defines. +// +// Parameters: +// call - The call node +// +// Returns: +// A tree representing the address of a local. +// +GenTreeLclVarCommon* Compiler::gtCallGetDefinedAsyncResumedLclAddr(GenTreeCall* call) +{ + if (!call->IsAsync()) + { + return nullptr; + } + + CallArg* arg = call->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedDef); + if (arg == nullptr) + { + return nullptr; + } + + GenTree* node = arg->GetNode(); + assert(node->OperIs(GT_LCL_ADDR) && lvaGetDesc(node->AsLclVarCommon())->IsDefinedViaAddress()); + return node->AsLclVarCommon(); +} + //------------------------------------------------------------------------ // ParseArrayAddress: Rehydrate the array and index expression from ARR_ADDR. // diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index b7c0feca6df7cb..775339048a6af1 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -7429,11 +7429,15 @@ void Compiler::impInheritAsyncContextsFromInliner(GenTreeCall* call) return; } - GenTreeCall* inlCall = impInlineInfo->iciCall; - CallArg* execArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncExecutionContext); - CallArg* syncArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncSynchronizationContext); + GenTreeCall* inlCall = impInlineInfo->iciCall; + CallArg* resumedUseArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedUse); + CallArg* resumedDefArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedDef); + CallArg* execArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncExecutionContext); + CallArg* syncArg = inlCall->gtArgs.FindWellKnownArg(WellKnownArg::AsyncSynchronizationContext); + assert((resumedUseArg == nullptr) == (resumedDefArg == nullptr)); + assert((resumedDefArg == nullptr) == (execArg == nullptr)); assert((execArg == nullptr) == (syncArg == nullptr)); - if ((execArg == nullptr) || (syncArg == nullptr)) + if (resumedUseArg == nullptr) { // Caller also has no async contexts handling return; @@ -7445,14 +7449,20 @@ void Compiler::impInheritAsyncContextsFromInliner(GenTreeCall* call) // caller's context here. It means we do not need to worry about switching // into the caller's context when the inlinee is returning to the caller // after the await. - assert(execArg->GetNode()->OperIs(GT_LCL_VAR) && syncArg->GetNode()->OperIs(GT_LCL_VAR)); - JITDUMP("Inheriting contexts [%06u] and [%06u] from caller node\n", dspTreeID(execArg->GetNode()), + assert(resumedUseArg->GetNode()->OperIs(GT_LCL_VAR) && resumedDefArg->GetNode()->OperIs(GT_LCL_ADDR) && + execArg->GetNode()->OperIs(GT_LCL_VAR) && syncArg->GetNode()->OperIs(GT_LCL_VAR)); + JITDUMP("Inheriting resumed use [%06u], resumed def [%06u], and contexts [%06u] and [%06u] from caller node\n", + dspTreeID(resumedUseArg->GetNode()), dspTreeID(resumedDefArg->GetNode()), dspTreeID(execArg->GetNode()), dspTreeID(syncArg->GetNode())); - GenTree* execNode = gtCloneExpr(execArg->GetNode()); - GenTree* syncNode = gtCloneExpr(syncArg->GetNode()); + GenTree* resumedUseNode = gtCloneExpr(resumedUseArg->GetNode()); + GenTree* resumedDefNode = gtCloneExpr(resumedDefArg->GetNode()); + GenTree* execNode = gtCloneExpr(execArg->GetNode()); + GenTree* syncNode = gtCloneExpr(syncArg->GetNode()); call->gtArgs.PushFront(this, NewCallArg::Primitive(syncNode).WellKnown(WellKnownArg::AsyncSynchronizationContext)); call->gtArgs.PushFront(this, NewCallArg::Primitive(execNode).WellKnown(WellKnownArg::AsyncExecutionContext)); + call->gtArgs.PushFront(this, NewCallArg::Primitive(resumedDefNode).WellKnown(WellKnownArg::AsyncResumedDef)); + call->gtArgs.PushFront(this, NewCallArg::Primitive(resumedUseNode).WellKnown(WellKnownArg::AsyncResumedUse)); } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index b4bf7874d19a5a..8c96b0d5c166be 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -102,22 +102,23 @@ class LocalSequencer final : public GenTreeVisitor } //------------------------------------------------------------------- - // SequenceCall: Post-process a call that may define a local. + // SequenceCall: Post-process a call that may define locals. // // Arguments: // call - the call // // Remarks: - // calls may also define a local that we would like to see - // after all other operands of the call have been evaluated. + // calls may also define locals that we would like to see after all + // other operands of the call have been evaluated. // void SequenceCall(GenTreeCall* call) { - if (call->IsOptimizingRetBufAsLocal()) - { - // Correct the point at which the definition of the retbuf local appears. - MoveNodeToEnd(m_compiler->gtCallGetDefinedRetBufLclAddr(call)); - } + auto moveToEnd = [&](GenTreeLclVarCommon* def) { + MoveNodeToEnd(def); + return GenTree::VisitResult::Continue; + }; + + call->VisitLocalDefNodes(m_compiler, moveToEnd); } //------------------------------------------------------------------- @@ -1495,32 +1496,47 @@ class LocalAddressVisitor final : public GenTreeVisitor GenTreeFlags defFlag = GTF_EMPTY; GenTreeCall* callUser = (user != nullptr) && user->IsCall() ? user->AsCall() : nullptr; bool escapeAddr = true; - if (m_compiler->opts.compJitOptimizeStructHiddenBuffer && (callUser != nullptr) && - m_compiler->IsValidLclAddr(lclNum, val.Offset())) - { - // We will only attempt this optimization for locals that do not - // later turn into indirections. - bool isSuitableLocal = - varTypeIsStruct(varDsc) && !m_compiler->lvaIsImplicitByRefLocal(lclNum) && - (!varDsc->lvIsStructField || !m_compiler->lvaIsImplicitByRefLocal(varDsc->lvParentLcl)); + if ((callUser != nullptr) && m_compiler->IsValidLclAddr(lclNum, val.Offset())) + { + unsigned defSize = UINT_MAX; + if (callUser->gtArgs.HasRetBuffer() && (val.Node() == callUser->gtArgs.GetRetBufferArg()->GetNode())) + { + // We will only attempt this optimization for locals that do not + // later turn into indirections. + bool isSuitableLocal = + m_compiler->opts.compJitOptimizeStructHiddenBuffer && varTypeIsStruct(varDsc) && + !m_compiler->lvaIsImplicitByRefLocal(lclNum) && + (!varDsc->lvIsStructField || !m_compiler->lvaIsImplicitByRefLocal(varDsc->lvParentLcl)); #ifdef TARGET_X86 - if (m_compiler->lvaIsArgAccessedViaVarArgsCookie(lclNum)) + if (m_compiler->lvaIsArgAccessedViaVarArgsCookie(lclNum)) + { + isSuitableLocal = false; + } +#endif // TARGET_X86 + + if (isSuitableLocal) + { + m_compiler->lvaSetHiddenBufferStructArg(lclNum); + callUser->gtCallMoreFlags |= GTF_CALL_M_RETBUFFARG_LCLOPT; + defSize = m_compiler->typGetObjLayout(callUser->gtRetClsHnd)->GetSize(); + } + } + else if (callUser->IsAsync()) { - isSuitableLocal = false; + CallArg* asyncResumedDef = callUser->gtArgs.FindWellKnownArg(WellKnownArg::AsyncResumedDef); + if ((asyncResumedDef != nullptr) && (val.Node() == asyncResumedDef->GetNode())) + { + defSize = 1; + } } -#endif // TARGET_X86 - if (isSuitableLocal && callUser->gtArgs.HasRetBuffer() && - (val.Node() == callUser->gtArgs.GetRetBufferArg()->GetNode())) + if (defSize != UINT_MAX) { - m_compiler->lvaSetHiddenBufferStructArg(lclNum); + INDEBUG(varDsc->SetDefinedViaAddress(true)); escapeAddr = false; - callUser->gtCallMoreFlags |= GTF_CALL_M_RETBUFFARG_LCLOPT; - defFlag = GTF_VAR_DEF; - - unsigned storeSize = m_compiler->typGetObjLayout(callUser->gtRetClsHnd)->GetSize(); + defFlag = GTF_VAR_DEF; - if (!m_compiler->IsEntireAccess(lclNum, val.Offset(), ValueSize(storeSize))) + if (!m_compiler->IsEntireAccess(lclNum, val.Offset(), ValueSize(defSize))) { defFlag |= GTF_VAR_USEASG; } diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index 7cd04b80336d2e..703e2f0b35c25d 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -2276,9 +2276,7 @@ void Compiler::lvaSetHiddenBufferStructArg(unsigned varNum) { LclVarDsc* varDsc = lvaGetDesc(varNum); -#ifdef DEBUG - varDsc->SetDefinedViaAddress(true); -#endif + INDEBUG(varDsc->SetDefinedViaAddress(true)); if (varDsc->lvPromoted) { diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 8897e68687c35c..9198ab6c529d0e 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -108,7 +108,7 @@ class Liveness bool* pStoreRemoved DEBUGARG(bool* treeModf)); void ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALARG_TP keepAliveVars); - bool IsTrackedRetBufferAddress(LIR::Range& range, GenTree* node); + bool IsTrackedCallDefinition(LIR::Range& range, GenTree* node); bool TryRemoveDeadStoreLIR(GenTree* store, GenTreeLclVarCommon* lclNode, BasicBlock* block); bool TryRemoveNonLocalLIR(GenTree* node, LIR::Range* blockRange); bool CanUncontainOrRemoveOperands(GenTree* node); @@ -743,9 +743,9 @@ void Liveness::PerNodeLocalVarLiveness(GenTree* tree) case GT_LCL_ADDR: if (TLiveness::IsLIR) { - // If this is a definition of a retbuf then we process it as - // part of the GT_CALL node. - if (IsTrackedRetBufferAddress(LIR::AsRange(m_compiler->compCurBB), tree)) + // If this is a call definition then we process it as part of + // the GT_CALL node. + if (IsTrackedCallDefinition(LIR::AsRange(m_compiler->compCurBB), tree)) { break; } @@ -2342,12 +2342,11 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR } else { - // For LCL_ADDRs that are defined by being passed as a - // retbuf we will handle them when we get to the call. We - // cannot consider them to be defined at the point of the - // LCL_ADDR since there may be uses between the LCL_ADDR - // and call. - if (IsTrackedRetBufferAddress(blockRange, node)) + // For LCL_ADDRs that are definitions for the call we will + // handle them when we get to the call. We cannot consider + // them to be defined at the point of the LCL_ADDR since + // there may be uses between the LCL_ADDR and call. + if (IsTrackedCallDefinition(blockRange, node)) { break; } @@ -2549,15 +2548,15 @@ void Liveness::ComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VAR } //--------------------------------------------------------------------- -// IsTrackedRetBufferAddress - given a LCL_ADDR node, check if it is the -// return buffer definition of a call. +// IsTrackedCallDefinition - given a LCL_ADDR node, check if it is an +// extra definition of a call. // // Arguments // range - the block range containing the LCL_ADDR // node - the LCL_ADDR // template -bool Liveness::IsTrackedRetBufferAddress(LIR::Range& range, GenTree* node) +bool Liveness::IsTrackedCallDefinition(LIR::Range& range, GenTree* node) { assert(node->OperIs(GT_LCL_ADDR)); if ((node->gtFlags & GTF_VAR_DEF) == 0) @@ -2584,7 +2583,11 @@ bool Liveness::IsTrackedRetBufferAddress(LIR::Range& range, GenTree* if (curNode->IsCall()) { - return m_compiler->gtCallGetDefinedRetBufLclAddr(curNode->AsCall()) == node; + auto visit = [=](GenTree* callDef) { + return node == callDef ? GenTree::VisitResult::Abort : GenTree::VisitResult::Continue; + }; + + return curNode->VisitLocalDefNodes(m_compiler, visit) == GenTree::VisitResult::Abort; } } while (curNode->OperIs(GT_FIELD_LIST) || curNode->OperIsPutArg()); diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 39e2c9f0eea43e..218aa03c423de9 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -1553,10 +1553,10 @@ void CallArgs::EvalArgsToTemps(Compiler* comp, GenTreeCall* call) arg.SetEarlyNode(setupArg); call->gtFlags |= setupArg->gtFlags & GTF_SIDE_EFFECT; - // Make sure we do not break recognition of retbuf-as-local - // optimization here. If this is hit it indicates that we are - // unnecessarily creating temps for some ret buf addresses, and - // gtCallGetDefinedRetBufLclAddr relies on this not to happen. + // Make sure we do not break recognition of defs optimization here. + // If this is hit it indicates that we are unnecessarily creating + // temps for some ret buf addresses, and call defs rely on this not + // to happen. noway_assert((arg.GetWellKnownArg() != WellKnownArg::RetBuffer) || !call->IsOptimizingRetBufAsLocal()); } diff --git a/src/coreclr/jit/promotionliveness.cpp b/src/coreclr/jit/promotionliveness.cpp index e137caab827500..8ba68f7711c5ca 100644 --- a/src/coreclr/jit/promotionliveness.cpp +++ b/src/coreclr/jit/promotionliveness.cpp @@ -257,12 +257,25 @@ unsigned PromotionLiveness::GetSizeOfStructLocal(Statement* stmt, GenTreeLclVarC { if (lcl->OperIs(GT_LCL_ADDR)) { - // Retbuf definition. Find the definition size from the - // containing call. + // LCL_ADDR definition. Currently we only have calls that define via + // LCL_ADDRs. Find the definition size from the containing call. Compiler::FindLinkData data = m_compiler->gtFindLink(stmt, lcl); - assert((data.parent != nullptr) && data.parent->IsCall() && - (m_compiler->gtCallGetDefinedRetBufLclAddr(data.parent->AsCall()) == lcl)); - return m_compiler->typGetObjLayout(data.parent->AsCall()->gtRetClsHnd)->GetSize(); + assert((data.parent != nullptr) && data.parent->IsCall()); + + unsigned defSize = UINT_MAX; + auto findDef = [&](const LocalDef& def) { + if (def.Def == lcl) + { + defSize = def.Size.GetExact(); + return GenTree::VisitResult::Abort; + } + + return GenTree::VisitResult::Continue; + }; + + GenTree::VisitResult result = data.parent->VisitLocalDefs(m_compiler, findDef); + assert(result == GenTree::VisitResult::Abort); + return defSize; } return lcl->GetLayout(m_compiler)->GetSize(); diff --git a/src/coreclr/jit/wellknownargs.h b/src/coreclr/jit/wellknownargs.h index a963226292cbbc..56e3ad962095dd 100644 --- a/src/coreclr/jit/wellknownargs.h +++ b/src/coreclr/jit/wellknownargs.h @@ -38,6 +38,8 @@ WELL_KNOWN_ARG(StackArrayLocal, "&lcl arr", false, false) WELL_KNOWN_ARG(RuntimeMethodHandle, "meth hnd", false, false) WELL_KNOWN_ARG(AsyncExecutionContext, "exec ctx", false, false) WELL_KNOWN_ARG(AsyncSynchronizationContext, "sync ctx", false, false) +WELL_KNOWN_ARG(AsyncResumedUse, "resumed", false, false) +WELL_KNOWN_ARG(AsyncResumedDef, "resumed def", false, false) WELL_KNOWN_ARG(WasmShadowStackPointer, "wasm sp", false, false) WELL_KNOWN_ARG(WasmPortableEntryPoint, "wasm pep", false, false)