20#define DEBUG_TYPE "memory-profile-info"
25 "memprof-lifetime-access-density-cold-threshold",
cl::init(0.05),
27 cl::desc(
"The threshold the lifetime access density (accesses per byte per "
28 "lifetime sec) must be under to consider an allocation cold"));
34 cl::desc(
"The average lifetime (s) for an allocation to be considered "
40 "memprof-min-ave-lifetime-access-density-hot-threshold",
cl::init(1000),
42 cl::desc(
"The minimum TotalLifetimeAccessDensity / AllocCount for an "
43 "allocation to be considered hot"));
47 cl::desc(
"Enable use of hot hints (only supported for "
48 "unambigously hot allocations)"));
52 cl::desc(
"Report total allocation sizes of hinted allocations"));
59 cl::desc(
"Keep all non-cold contexts (increases cloning overheads)"));
66 if (((
float)TotalLifetimeAccessDensity) / AllocCount / 100 <
69 && ((
float)TotalLifetime) / AllocCount >=
71 return AllocationType::Cold;
76 ((
float)TotalLifetimeAccessDensity) / AllocCount / 100 >
78 return AllocationType::Hot;
80 return AllocationType::NotCold;
106 auto *MDS = dyn_cast<MDString>(MIB->
getOperand(1));
108 if (MDS->getString() ==
"cold") {
109 return AllocationType::Cold;
110 }
else if (MDS->getString() ==
"hot") {
111 return AllocationType::Hot;
113 return AllocationType::NotCold;
118 case AllocationType::NotCold:
121 case AllocationType::Cold:
124 case AllocationType::Hot:
128 assert(
false &&
"Unexpected alloc type");
142 assert(NumAllocTypes != 0);
143 return NumAllocTypes == 1;
148 std::vector<ContextTotalSize> ContextSizeInfo) {
150 CallStackTrieNode *Curr =
nullptr;
151 for (
auto StackId : StackIds) {
156 assert(AllocStackId == StackId);
159 AllocStackId = StackId;
160 Alloc =
new CallStackTrieNode(
AllocType);
166 CallStackTrieNode *Prev =
nullptr;
167 auto Next = Curr->Callers.find(StackId);
168 if (Next != Curr->Callers.end()) {
175 Prev->DeepestAmbiguousAllocType =
false;
179 auto *New =
new CallStackTrieNode(
AllocType);
180 Curr->Callers[StackId] = New;
184 Curr->ContextSizeInfo.insert(Curr->ContextSizeInfo.end(),
185 ContextSizeInfo.begin(), ContextSizeInfo.end());
193 for (
const auto &MIBStackIter : StackMD->
operands()) {
194 auto *StackId = mdconst::dyn_extract<ConstantInt>(MIBStackIter);
196 CallStack.push_back(StackId->getZExtValue());
198 std::vector<ContextTotalSize> ContextSizeInfo;
205 mdconst::dyn_extract<ConstantInt>(ContextSizePair->
getOperand(0))
208 mdconst::dyn_extract<ConstantInt>(ContextSizePair->
getOperand(1))
210 ContextSizeInfo.push_back({FullStackId, TotalSize});
223 if (!ContextSizeInfo.
empty()) {
224 for (
const auto &[FullStackId, TotalSize] : ContextSizeInfo) {
229 auto *ContextSizeMD =
MDNode::get(Ctx, {FullStackIdMD, TotalSizeMD});
236void CallStackTrie::collectContextSizeInfo(
237 CallStackTrieNode *
Node, std::vector<ContextTotalSize> &ContextSizeInfo) {
238 ContextSizeInfo.insert(ContextSizeInfo.end(),
Node->ContextSizeInfo.begin(),
239 Node->ContextSizeInfo.end());
240 for (
auto &Caller :
Node->Callers)
241 collectContextSizeInfo(
Caller.second, ContextSizeInfo);
244void CallStackTrie::convertHotToNotCold(CallStackTrieNode *
Node) {
249 for (
auto &Caller :
Node->Callers)
250 convertHotToNotCold(
Caller.second);
256bool CallStackTrie::buildMIBNodes(CallStackTrieNode *
Node,
LLVMContext &Ctx,
257 std::vector<uint64_t> &MIBCallStack,
258 std::vector<Metadata *> &MIBNodes,
259 bool CalleeHasAmbiguousCallerContext,
260 bool &CalleeDeepestAmbiguousAllocType) {
278 std::vector<ContextTotalSize> ContextSizeInfo;
279 collectContextSizeInfo(
Node, ContextSizeInfo);
286 CalleeDeepestAmbiguousAllocType =
false;
293 if (!
Node->Callers.empty()) {
294 bool NodeHasAmbiguousCallerContext =
Node->Callers.size() > 1;
295 bool AddedMIBNodesForAllCallerContexts =
true;
296 for (
auto &Caller :
Node->Callers) {
297 MIBCallStack.push_back(
Caller.first);
298 AddedMIBNodesForAllCallerContexts &= buildMIBNodes(
299 Caller.second, Ctx, MIBCallStack, MIBNodes,
300 NodeHasAmbiguousCallerContext,
Node->DeepestAmbiguousAllocType);
302 MIBCallStack.pop_back();
304 if (AddedMIBNodesForAllCallerContexts)
308 assert(!NodeHasAmbiguousCallerContext);
321 if (!CalleeHasAmbiguousCallerContext)
323 std::vector<ContextTotalSize> ContextSizeInfo;
324 collectContextSizeInfo(
Node, ContextSizeInfo);
334 std::vector<ContextTotalSize> ContextSizeInfo;
335 collectContextSizeInfo(Alloc, ContextSizeInfo);
336 for (
const auto &[FullStackId, TotalSize] : ContextSizeInfo) {
337 errs() <<
"MemProf hinting: Total size for full allocation context hash "
338 << FullStackId <<
" and " << Descriptor <<
" alloc type "
361 convertHotToNotCold(Alloc);
370 std::vector<uint64_t> MIBCallStack;
371 MIBCallStack.push_back(AllocStackId);
372 std::vector<Metadata *> MIBNodes;
373 assert(!Alloc->Callers.empty() &&
"addCallStack has not been called yet");
380 bool DeepestAmbiguousAllocType =
true;
381 if (buildMIBNodes(Alloc, Ctx, MIBCallStack, MIBNodes,
383 DeepestAmbiguousAllocType)) {
384 assert(MIBCallStack.size() == 1 &&
385 "Should only be left with Alloc's location in stack");
404 Iter =
End ?
N->op_end() :
N->op_begin();
418 return mdconst::dyn_extract<ConstantInt>(
N->operands().back())
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
cl::opt< float > MemProfLifetimeAccessDensityColdThreshold("memprof-lifetime-access-density-cold-threshold", cl::init(0.05), cl::Hidden, cl::desc("The threshold the lifetime access density (accesses per byte per " "lifetime sec) must be under to consider an allocation cold"))
cl::opt< bool > MemProfKeepAllNotColdContexts("memprof-keep-all-not-cold-contexts", cl::init(false), cl::Hidden, cl::desc("Keep all non-cold contexts (increases cloning overheads)"))
cl::opt< unsigned > MemProfMinAveLifetimeAccessDensityHotThreshold("memprof-min-ave-lifetime-access-density-hot-threshold", cl::init(1000), cl::Hidden, cl::desc("The minimum TotalLifetimeAccessDensity / AllocCount for an " "allocation to be considered hot"))
cl::opt< bool > MemProfUseHotHints("memprof-use-hot-hints", cl::init(false), cl::Hidden, cl::desc("Enable use of hot hints (only supported for " "unambigously hot allocations)"))
cl::opt< bool > MemProfReportHintedSizes("memprof-report-hinted-sizes", cl::init(false), cl::Hidden, cl::desc("Report total allocation sizes of hinted allocations"))
static MDNode * createMIBNode(LLVMContext &Ctx, ArrayRef< uint64_t > MIBCallStack, AllocationType AllocType, ArrayRef< ContextTotalSize > ContextSizeInfo)
cl::opt< unsigned > MemProfAveLifetimeColdThreshold("memprof-ave-lifetime-cold-threshold", cl::init(200), cl::Hidden, cl::desc("The average lifetime (s) for an allocation to be considered " "cold"))
static void addAllocTypeAttribute(LLVMContext &Ctx, CallBase *CI, AllocationType AllocType)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
static MDNode * getMergedCallsiteMetadata(MDNode *A, MDNode *B)
const MDOperand & getOperand(unsigned I) const
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDNode * getMergedMemProfMetadata(MDNode *A, MDNode *B)
static MDString * get(LLVMContext &Context, StringRef Str)
void push_back(Metadata *MD)
Append an element to the tuple. This will resize the node.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt64Ty(LLVMContext &C)
LLVMContext & getContext() const
All values hold a context through their type.
void addCallStack(AllocationType AllocType, ArrayRef< uint64_t > StackIds, std::vector< ContextTotalSize > ContextSizeInfo={})
Add a call stack context with the given allocation type to the Trie.
void addSingleAllocTypeAttribute(CallBase *CI, AllocationType AT, StringRef Descriptor)
Add an attribute for the given allocation type to the call instruction.
bool buildAndAttachMIBMetadata(CallBase *CI)
Build and attach the minimal necessary MIB metadata.
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
initializer< Ty > init(const Ty &Val)
MDNode * buildCallstackMetadata(ArrayRef< uint64_t > CallStack, LLVMContext &Ctx)
Build callstack metadata from the provided list of call stack ids.
AllocationType getAllocType(uint64_t TotalLifetimeAccessDensity, uint64_t AllocCount, uint64_t TotalLifetime)
Return the allocation type for a given set of memory profile values.
AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
bool hasSingleAllocType(uint8_t AllocTypes)
True if the AllocTypes bitmask contains just a single type.
std::string getAllocTypeAttributeString(AllocationType Type)
Returns the string to use in attributes with the given type.
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
This is an optimization pass for GlobalISel generic memory operations.
int popcount(T Value) noexcept
Count the number of set bits in a value.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
CallStackIterator(const NodeT *N, bool End)