Skip to content

Commit 30082a4

Browse files
AndyAyersMSEgorBo
AndyAyersMS
and
EgorBo
authored
JIT: save pgo data in inline context, use it for call optimization (#116241)
Fix #116223 Save profile data (schema, schema size, and data) in the inline context. Always store the inline context in GT_CALL nodes. Use this to find the correct PGO data to consult when optimizing a call (in particular, for late cast expansions). Also, dump likely class in gtDispTree. Co-authored-by: EgorBo <[email protected]>
1 parent c630f27 commit 30082a4

File tree

8 files changed

+116
-33
lines changed

8 files changed

+116
-33
lines changed

‎src/coreclr/jit/compiler.cpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2471,6 +2471,7 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
24712471
fgPgoDisabled = true;
24722472
}
24732473
}
2474+
#endif // DEBUG
24742475

24752476
// A successful result implies a non-NULL fgPgoSchema
24762477
//
@@ -2490,6 +2491,11 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
24902491
break;
24912492
}
24922493
}
2494+
2495+
// Stash pointers to PGO info on the context so
2496+
// we can access contextually it later.
2497+
//
2498+
compInlineContext->SetPgoInfo(PgoInfo(this));
24932499
}
24942500

24952501
// A failed result implies a NULL fgPgoSchema
@@ -2499,7 +2505,6 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
24992505
{
25002506
assert(fgPgoSchema == nullptr);
25012507
}
2502-
#endif // DEBUG
25032508
}
25042509

25052510
bool enableInliningMethodsWithEH = JitConfig.JitInlineMethodsWithEH() > 0;

‎src/coreclr/jit/compiler.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7540,7 +7540,8 @@ class Compiler
75407540
CORINFO_CLASS_HANDLE* classGuesses,
75417541
CORINFO_METHOD_HANDLE* methodGuesses,
75427542
int* candidatesCount,
7543-
unsigned* likelihoods);
7543+
unsigned* likelihoods,
7544+
bool verboseLogging = true);
75447545

75457546
void considerGuardedDevirtualization(GenTreeCall* call,
75467547
IL_OFFSET ilOffset,

‎src/coreclr/jit/gentree.cpp

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8346,9 +8346,11 @@ GenTreeCall* Compiler::gtNewCallNode(gtCallTypes callType,
83468346
// These get updated after call node is built.
83478347
node->gtInlineObservation = InlineObservation::CALLEE_UNUSED_INITIAL;
83488348
node->gtRawILOffset = BAD_IL_OFFSET;
8349-
node->gtInlineContext = compInlineContext;
8349+
83508350
#endif
83518351

8352+
node->gtInlineContext = compInlineContext;
8353+
83528354
// Spec: Managed Retval sequence points needs to be generated while generating debug info for debuggable code.
83538355
//
83548356
// Implementation note: if not generating MRV info genCallSite2ILOffsetMap will be NULL and
@@ -9951,9 +9953,10 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree)
99519953
#if defined(DEBUG)
99529954
copy->gtInlineObservation = tree->gtInlineObservation;
99539955
copy->gtRawILOffset = tree->gtRawILOffset;
9954-
copy->gtInlineContext = tree->gtInlineContext;
99559956
#endif
99569957

9958+
copy->gtInlineContext = tree->gtInlineContext;
9959+
99579960
copy->CopyOtherRegFlags(tree);
99589961

99599962
// We keep track of the number of no return calls, so if we've cloned
@@ -13012,6 +13015,20 @@ void Compiler::gtDispTree(GenTree* tree,
1301213015
}
1301313016
}
1301413017

13018+
// Dump profile if any
13019+
if (call->IsHelperCall() && impIsCastHelperMayHaveProfileData(eeGetHelperNum(call->gtCallMethHnd)))
13020+
{
13021+
CORINFO_CLASS_HANDLE likelyClasses[MAX_GDV_TYPE_CHECKS] = {};
13022+
unsigned likelyLikelihoods[MAX_GDV_TYPE_CHECKS] = {};
13023+
int likelyClassCount = 0;
13024+
pickGDV(call, call->gtCastHelperILOffset, false, likelyClasses, nullptr, &likelyClassCount,
13025+
likelyLikelihoods, false);
13026+
if (likelyClassCount > 0)
13027+
{
13028+
printf(" (%d%% likely '%s')", likelyLikelihoods[0], eeGetClassName(likelyClasses[0]));
13029+
}
13030+
}
13031+
1301513032
gtDispCommonEndLine(tree);
1301613033

1301713034
if (!topOnly)

‎src/coreclr/jit/gentree.h

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5660,12 +5660,9 @@ struct GenTreeCall final : public GenTree
56605660

56615661
// IL offset of the call wrt its parent method.
56625662
IL_OFFSET gtRawILOffset;
5663+
#endif // defined(DEBUG)
56635664

5664-
// In DEBUG we report even non inline candidates in the inline tree in
5665-
// fgNoteNonInlineCandidate. We need to keep around the inline context for
5666-
// this as normally it's part of the candidate info.
56675665
class InlineContext* gtInlineContext;
5668-
#endif // defined(DEBUG)
56695666

56705667
bool IsHelperCall() const
56715668
{

‎src/coreclr/jit/helperexpansion.cpp

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1942,10 +1942,6 @@ enum class TypeCheckPassedAction
19421942
CallHelper_AlwaysThrows,
19431943
};
19441944

1945-
// Some arbitrary limit on the number of guesses we can make
1946-
// The actual number of guesses is usually much smaller
1947-
#define MAX_CAST_GUESSES 8
1948-
19491945
//------------------------------------------------------------------------------
19501946
// PickCandidatesForTypeCheck: picks classes to use as fast type checks against
19511947
// the object being casted. The function also defines the strategy to follow
@@ -1954,7 +1950,7 @@ enum class TypeCheckPassedAction
19541950
// Arguments:
19551951
// comp - Compiler instance
19561952
// castHelper - Cast helper call to expand
1957-
// candidates - [out] Classes (guesses) to use in the fast path (up to MAX_CAST_GUESSES)
1953+
// candidates - [out] Classes (guesses) to use in the fast path (up to MAX_GDV_TYPE_CHECKS)
19581954
// commonCls - [out] Common denominator class for the fast and the fallback paths.
19591955
// likelihoods - [out] Likelihoods of successful type checks [0..100]
19601956
// typeCheckFailed - [out] Action to perform if the type check fails
@@ -2160,9 +2156,9 @@ static int PickCandidatesForTypeCheck(Compiler* comp,
21602156
/////////////////////////////////////////////////////////////////////////////////////////////////////
21612157

21622158
// Let's re-use GDV's threshold on how many guesses we can make (can be 3 by default).
2163-
const int maxTypeChecks = min(comp->getGDVMaxTypeChecks(), MAX_CAST_GUESSES);
2159+
const int maxTypeChecks = min(comp->getGDVMaxTypeChecks(), MAX_GDV_TYPE_CHECKS);
21642160

2165-
CORINFO_CLASS_HANDLE exactClasses[MAX_CAST_GUESSES] = {};
2161+
CORINFO_CLASS_HANDLE exactClasses[MAX_GDV_TYPE_CHECKS] = {};
21662162
const int numExactClasses = comp->info.compCompHnd->getExactClasses(castToCls, maxTypeChecks, exactClasses);
21672163
bool allTrulyExact = true;
21682164
for (int i = 0; i < numExactClasses; i++)
@@ -2234,9 +2230,9 @@ static int PickCandidatesForTypeCheck(Compiler* comp,
22342230
// 3) Consult with PGO data
22352231
/////////////////////////////////////////////////////////////////////////////////////////////////////
22362232

2237-
CORINFO_CLASS_HANDLE likelyClasses[MAX_CAST_GUESSES] = {};
2238-
unsigned likelyLikelihoods[MAX_CAST_GUESSES] = {};
2239-
int likelyClassCount = 0;
2233+
CORINFO_CLASS_HANDLE likelyClasses[MAX_GDV_TYPE_CHECKS] = {};
2234+
unsigned likelyLikelihoods[MAX_GDV_TYPE_CHECKS] = {};
2235+
int likelyClassCount = 0;
22402236
comp->pickGDV(castHelper, castHelper->gtCastHelperILOffset, false, likelyClasses, nullptr, &likelyClassCount,
22412237
likelyLikelihoods);
22422238

@@ -2364,8 +2360,8 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
23642360
TypeCheckFailedAction typeCheckFailedAction;
23652361
TypeCheckPassedAction typeCheckPassedAction;
23662362
CORINFO_CLASS_HANDLE commonCls;
2367-
CORINFO_CLASS_HANDLE expectedExactClasses[MAX_CAST_GUESSES] = {};
2368-
unsigned likelihoods[MAX_CAST_GUESSES] = {};
2363+
CORINFO_CLASS_HANDLE expectedExactClasses[MAX_GDV_TYPE_CHECKS] = {};
2364+
unsigned likelihoods[MAX_GDV_TYPE_CHECKS] = {};
23692365

23702366
const int numOfCandidates = PickCandidatesForTypeCheck(this, call, expectedExactClasses, &commonCls, likelihoods,
23712367
&typeCheckFailedAction, &typeCheckPassedAction);
@@ -2450,8 +2446,8 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt,
24502446
// Block 2: typeCheckBb(s)
24512447
// TODO-InlineCast: if likelyCls == expectedCls we can consider saving to a local to re-use.
24522448

2453-
BasicBlock* typeChecksBbs[MAX_CAST_GUESSES] = {};
2454-
BasicBlock* lastTypeCheckBb = nullcheckBb;
2449+
BasicBlock* typeChecksBbs[MAX_GDV_TYPE_CHECKS] = {};
2450+
BasicBlock* lastTypeCheckBb = nullcheckBb;
24552451
for (int candidateId = 0; candidateId < numOfCandidates; candidateId++)
24562452
{
24572453
const CORINFO_CLASS_HANDLE expectedCls = expectedExactClasses[candidateId];

‎src/coreclr/jit/importercalls.cpp

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6748,24 +6748,30 @@ void Compiler::addFatPointerCandidate(GenTreeCall* call)
67486748
// methodGuesses - [out] the methods to guess for (mutually exclusive with classGuess)
67496749
// candidatesCount - [out] number of guesses
67506750
// likelihoods - [out] estimates of the likelihoods that the guesses will succeed
6751+
// verboseLogging - whether or not to do verbose logging
67516752
//
67526753
void Compiler::pickGDV(GenTreeCall* call,
67536754
IL_OFFSET ilOffset,
67546755
bool isInterface,
67556756
CORINFO_CLASS_HANDLE* classGuesses,
67566757
CORINFO_METHOD_HANDLE* methodGuesses,
67576758
int* candidatesCount,
6758-
unsigned* likelihoods)
6759+
unsigned* likelihoods,
6760+
bool verboseLogging)
67596761
{
67606762
*candidatesCount = 0;
67616763

6764+
// Get the relevant pgo info for this call
6765+
//
6766+
PgoInfo pgoInfo(call->gtInlineContext);
6767+
67626768
const int maxLikelyClasses = MAX_GDV_TYPE_CHECKS;
67636769
LikelyClassMethodRecord likelyClasses[maxLikelyClasses];
67646770
unsigned numberOfClasses = 0;
67656771
if (call->IsVirtualStub() || call->IsVirtualVtable() || call->IsHelperCall())
67666772
{
6767-
numberOfClasses =
6768-
getLikelyClasses(likelyClasses, maxLikelyClasses, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset);
6773+
numberOfClasses = getLikelyClasses(likelyClasses, maxLikelyClasses, pgoInfo.PgoSchema, pgoInfo.PgoSchemaCount,
6774+
pgoInfo.PgoData, ilOffset);
67696775
}
67706776

67716777
const int maxLikelyMethods = MAX_GDV_TYPE_CHECKS;
@@ -6781,18 +6787,21 @@ void Compiler::pickGDV(GenTreeCall* call,
67816787
if (!IsAot() && (call->IsVirtualVtable() || call->IsDelegateInvoke()))
67826788
{
67836789
assert(!call->IsHelperCall());
6784-
numberOfMethods =
6785-
getLikelyMethods(likelyMethods, maxLikelyMethods, fgPgoSchema, fgPgoSchemaCount, fgPgoData, ilOffset);
6790+
numberOfMethods = getLikelyMethods(likelyMethods, maxLikelyMethods, pgoInfo.PgoSchema, pgoInfo.PgoSchemaCount,
6791+
pgoInfo.PgoData, ilOffset);
67866792
}
67876793

67886794
if ((numberOfClasses < 1) && (numberOfMethods < 1))
67896795
{
6790-
JITDUMP("No likely class or method, sorry\n");
6796+
if (verboseLogging)
6797+
{
6798+
JITDUMP("No likely class or method, sorry\n");
6799+
}
67916800
return;
67926801
}
67936802

67946803
#ifdef DEBUG
6795-
if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfClasses > 0))
6804+
if ((verbose || JitConfig.EnableExtraSuperPmiQueries()) && (numberOfClasses > 0) && verboseLogging)
67966805
{
67976806
JITDUMP("Likely classes for call [%06u]", dspTreeID(call));
67986807
if (!call->IsHelperCall())
@@ -6962,8 +6971,12 @@ void Compiler::pickGDV(GenTreeCall* call,
69626971
classGuesses[guessIdx] = (CORINFO_CLASS_HANDLE)likelyClasses[guessIdx].handle;
69636972
likelihoods[guessIdx] = likelyClasses[guessIdx].likelihood;
69646973
*candidatesCount = *candidatesCount + 1;
6965-
JITDUMP("Accepting type %s with likelihood %u as a candidate\n", eeGetClassName(classGuesses[guessIdx]),
6966-
likelihoods[guessIdx])
6974+
6975+
if (verboseLogging)
6976+
{
6977+
JITDUMP("Accepting type %s with likelihood %u as a candidate\n",
6978+
eeGetClassName(classGuesses[guessIdx]), likelihoods[guessIdx])
6979+
}
69676980
}
69686981
else
69696982
{
@@ -6986,8 +6999,11 @@ void Compiler::pickGDV(GenTreeCall* call,
69866999
return;
69877000
}
69887001

6989-
JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n",
6990-
call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold);
7002+
if (verboseLogging)
7003+
{
7004+
JITDUMP("Not guessing for method; likelihood is below %s call threshold %u\n",
7005+
call->IsDelegateInvoke() ? "delegate" : "virtual", likelihoodThreshold);
7006+
}
69917007
}
69927008
}
69937009

‎src/coreclr/jit/inline.cpp

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@ InlineContext::InlineContext(InlineStrategy* strategy)
332332
, m_Code(nullptr)
333333
, m_Callee(nullptr)
334334
, m_RuntimeContext(nullptr)
335+
, m_PgoInfo()
335336
, m_ILSize(0)
336337
, m_ImportedILSize(0)
337338
, m_ActualCallOffset(BAD_IL_OFFSET)
@@ -1823,3 +1824,22 @@ bool InlineStrategy::IsInliningDisabled()
18231824

18241825
#endif // defined(DEBUG)
18251826
}
1827+
1828+
PgoInfo::PgoInfo()
1829+
{
1830+
PgoSchema = nullptr;
1831+
PgoSchemaCount = 0;
1832+
PgoData = nullptr;
1833+
}
1834+
1835+
PgoInfo::PgoInfo(Compiler* compiler)
1836+
{
1837+
PgoSchema = compiler->fgPgoSchema;
1838+
PgoSchemaCount = compiler->fgPgoSchemaCount;
1839+
PgoData = compiler->fgPgoData;
1840+
}
1841+
1842+
PgoInfo::PgoInfo(InlineContext* context)
1843+
{
1844+
*this = context->GetPgoInfo();
1845+
}

‎src/coreclr/jit/inline.h

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -715,6 +715,21 @@ struct InlineInfo
715715
BasicBlock* iciBlock; // The basic block iciStmt is in.
716716
};
717717

718+
//------------------------------------------------------------------------
719+
// PgoInfo
720+
// Schema and data for a method's PGO data.
721+
//
722+
struct PgoInfo
723+
{
724+
PgoInfo();
725+
PgoInfo(Compiler* compiler);
726+
PgoInfo(InlineContext* inlineContext);
727+
728+
ICorJitInfo::PgoInstrumentationSchema* PgoSchema; // pgo schema for method
729+
BYTE* PgoData; // pgo data for the method
730+
unsigned PgoSchemaCount; // count of schema elements
731+
};
732+
718733
// InlineContext tracks the inline history in a method.
719734
//
720735
// Notes:
@@ -870,6 +885,21 @@ class InlineContext
870885
}
871886
#endif
872887

888+
const PgoInfo& GetPgoInfo()
889+
{
890+
return m_PgoInfo;
891+
}
892+
893+
void SetPgoInfo(const PgoInfo& info)
894+
{
895+
m_PgoInfo = info;
896+
}
897+
898+
bool HasPgoInfo() const
899+
{
900+
return (m_PgoInfo.PgoSchema != nullptr) && (m_PgoInfo.PgoSchemaCount > 0) && (m_PgoInfo.PgoData != nullptr);
901+
}
902+
873903
private:
874904
InlineContext(InlineStrategy* strategy);
875905

@@ -880,6 +910,7 @@ class InlineContext
880910
const BYTE* m_Code; // address of IL buffer for the method
881911
CORINFO_METHOD_HANDLE m_Callee; // handle to the method
882912
CORINFO_CONTEXT_HANDLE m_RuntimeContext; // handle to the exact context
913+
PgoInfo m_PgoInfo; // profile data
883914
unsigned m_ILSize; // size of IL buffer for the method
884915
unsigned m_ImportedILSize; // estimated size of imported IL
885916
ILLocation m_Location; // inlining statement location within parent

0 commit comments

Comments
 (0)