19 const DependencyGraph &DAG) {
20 auto Skip = [&DAG](
auto OpIt) {
21 auto *
I = dyn_cast<Instruction>((*OpIt).get());
22 return I ==
nullptr || DAG.getNode(
I) ==
nullptr;
24 while (OpIt != OpItE &&
Skip(OpIt))
31 if (!isa<MemDGNode>(N)) {
32 assert(OpIt != OpItE &&
"Can't dereference end iterator!");
33 return DAG->getNode(cast<Instruction>((
Value *)*OpIt));
38 return DAG->getNode(cast<Instruction>((
Value *)*OpIt));
40 assert(MemIt != cast<MemDGNode>(N)->MemPreds.end() &&
41 "Cant' dereference end iterator!");
47 if (!isa<MemDGNode>(N)) {
48 assert(OpIt != OpItE &&
"Already at end!");
51 OpIt = PredIterator::skipBadIt(OpIt, OpItE, *DAG);
59 OpIt = PredIterator::skipBadIt(OpIt, OpItE, *DAG);
63 assert(MemIt != cast<MemDGNode>(N)->MemPreds.end() &&
"Already at end!");
69 assert(DAG ==
Other.DAG &&
"Iterators of different DAGs!");
70 assert(N ==
Other.N &&
"Iterators of different nodes!");
71 return OpIt ==
Other.OpIt && MemIt ==
Other.MemIt;
77 SB->eraseFromBundle(
this);
89 static constexpr const unsigned Indent = 4;
90 for (
auto *Pred : MemPreds)
91 OS.indent(Indent) <<
"<-" << *Pred->getInstruction() <<
"\n";
103 I =
I->getNextNode();
106 return cast<MemDGNode>(DAG.
getNode(
I));
116 I =
I->getPrevNode();
119 return cast<MemDGNode>(DAG.
getNode(
I));
129 if (TopMemN ==
nullptr)
132 assert(BotMemN !=
nullptr &&
"TopMemN should be null too!");
137DependencyGraph::DependencyType
142 return DependencyType::ReadAfterWrite;
144 return DependencyType::WriteAfterWrite;
147 return DependencyType::WriteAfterRead;
149 if (isa<sandboxir::PHINode>(FromI) || isa<sandboxir::PHINode>(ToI))
150 return DependencyType::Control;
152 return DependencyType::Control;
155 return DependencyType::Other;
156 return DependencyType::None;
161 if (
auto *LI = dyn_cast<LoadInst>(
I))
162 return !LI->isUnordered();
163 if (
auto *SI = dyn_cast<StoreInst>(
I))
164 return !SI->isUnordered();
169 bool Is = IsOrdered(
I);
171 "An ordered instruction must be a MemDepCandidate!");
176 DependencyType DepType) {
177 std::optional<MemoryLocation> DstLocOpt =
183 "Expected a mem instr");
190 case DependencyType::ReadAfterWrite:
191 case DependencyType::WriteAfterWrite:
193 case DependencyType::WriteAfterRead:
200bool DependencyGraph::hasDep(Instruction *SrcI, Instruction *DstI) {
201 DependencyType RoughDepType = getRoughDepType(SrcI, DstI);
202 switch (RoughDepType) {
203 case DependencyType::ReadAfterWrite:
204 case DependencyType::WriteAfterWrite:
205 case DependencyType::WriteAfterRead:
206 return alias(SrcI, DstI, RoughDepType);
207 case DependencyType::Control:
213 case DependencyType::Other:
215 case DependencyType::None:
221void DependencyGraph::scanAndAddDeps(MemDGNode &DstN,
223 assert(isa<MemDGNode>(DstN) &&
224 "DstN is the mem dep destination, so it must be mem");
225 Instruction *DstI = DstN.getInstruction();
228 for (MemDGNode &SrcN :
reverse(SrcScanRange)) {
229 Instruction *SrcI = SrcN.getInstruction();
230 if (hasDep(SrcI, DstI))
231 DstN.addMemPred(&SrcN);
235void DependencyGraph::setDefUseUnscheduledSuccs(
244 for (Instruction &
I : NewInterval) {
245 for (Value *
Op :
I.operands()) {
246 auto *OpI = dyn_cast<Instruction>(
Op);
250 if (OpI->getParent() !=
I.getParent())
252 if (!NewInterval.contains(OpI))
257 ++OpN->UnscheduledSuccs;
262 bool NewIsAbove = DAGInterval.empty() || NewInterval.comesBefore(DAGInterval);
263 const auto &TopInterval = NewIsAbove ? NewInterval : DAGInterval;
264 const auto &BotInterval = NewIsAbove ? DAGInterval : NewInterval;
275 for (Instruction &BotI : BotInterval) {
278 if (BotN->scheduled())
280 for (Value *
Op : BotI.operands()) {
281 auto *OpI = dyn_cast<Instruction>(
Op);
287 if (!TopInterval.contains(OpI))
289 ++OpN->UnscheduledSuccs;
297 MemDGNode *LastMemN = dyn_cast<MemDGNode>(LastN);
301 if (
auto *MemN = dyn_cast<MemDGNode>(
N)) {
302 MemN->setPrevNode(LastMemN);
307 if (!DAGInterval.empty()) {
308 bool NewIsAbove = NewInterval.comesBefore(DAGInterval);
309 const auto &TopInterval = NewIsAbove ? NewInterval : DAGInterval;
310 const auto &BotInterval = NewIsAbove ? DAGInterval : NewInterval;
315 assert((LinkTopN ==
nullptr || LinkBotN ==
nullptr ||
316 LinkTopN->comesBefore(LinkBotN)) &&
318 if (LinkTopN !=
nullptr && LinkBotN !=
nullptr) {
319 LinkTopN->setNextNode(LinkBotN);
324 auto UnionIntvl = DAGInterval.getUnionInterval(NewInterval);
329 if (ChainTopN !=
nullptr && ChainBotN !=
nullptr) {
330 for (
auto *
N = ChainTopN->getNextNode(), *LastN = ChainTopN;
N !=
nullptr;
331 LastN =
N,
N =
N->getNextNode()) {
332 assert(
N == LastN->getNextNode() &&
"Bad chain!");
333 assert(
N->getPrevNode() == LastN &&
"Bad chain!");
339 setDefUseUnscheduledSuccs(NewInterval);
342MemDGNode *DependencyGraph::getMemDGNodeBefore(DGNode *
N,
bool IncludingN,
343 MemDGNode *SkipN)
const {
344 auto *
I =
N->getInstruction();
345 for (
auto *PrevI = IncludingN ?
I :
I->getPrevNode(); PrevI !=
nullptr;
346 PrevI = PrevI->getPrevNode()) {
348 if (PrevN ==
nullptr)
350 auto *PrevMemN = dyn_cast<MemDGNode>(PrevN);
351 if (PrevMemN !=
nullptr && PrevMemN != SkipN)
357MemDGNode *DependencyGraph::getMemDGNodeAfter(DGNode *
N,
bool IncludingN,
358 MemDGNode *SkipN)
const {
359 auto *
I =
N->getInstruction();
360 for (
auto *NextI = IncludingN ?
I :
I->getNextNode(); NextI !=
nullptr;
361 NextI = NextI->getNextNode()) {
363 if (NextN ==
nullptr)
365 auto *NextMemN = dyn_cast<MemDGNode>(NextN);
366 if (NextMemN !=
nullptr && NextMemN != SkipN)
372void DependencyGraph::notifyCreateInstr(Instruction *
I) {
377 if (!(DAGInterval.contains(
I) || DAGInterval.touches(
I)))
380 DAGInterval = DAGInterval.getUnionInterval({
I,
I});
382 auto *MemN = dyn_cast<MemDGNode>(
N);
385 if (MemN !=
nullptr) {
386 if (
auto *PrevMemN = getMemDGNodeBefore(MemN,
false)) {
387 PrevMemN->NextMemN = MemN;
388 MemN->PrevMemN = PrevMemN;
390 if (
auto *NextMemN = getMemDGNodeAfter(MemN,
false)) {
391 NextMemN->PrevMemN = MemN;
392 MemN->NextMemN = NextMemN;
397 if (DAGInterval.top()->comesBefore(
I)) {
400 scanAndAddDeps(*MemN, SrcInterval);
403 if (
I->comesBefore(DAGInterval.bottom())) {
405 for (MemDGNode &BelowN :
412void DependencyGraph::notifyMoveInstr(Instruction *
I,
const BBIterator &To) {
418 assert(!(To != BB->end() && &*To ==
I->getNextNode()) &&
419 !(To == BB->end() && std::next(
I->getIterator()) == BB->end()) &&
420 "Should not have been called if destination is same as origin.");
424 assert(To.getNodeParent() ==
I->getParent() &&
425 "TODO: We don't support movement across BBs!");
427 (To == std::next(DAGInterval.bottom()->getIterator()) ||
428 (To != BB->end() && std::next(To) == DAGInterval.top()->getIterator()) ||
429 (To != BB->end() && DAGInterval.contains(&*To))) &&
430 "TODO: To should be either within the DAGInterval or right "
434 auto OrigDAGInterval = DAGInterval;
437 DAGInterval.notifyMoveInstr(
I, To);
450 MemN->detachFromChain();
465 if (To == BB->end() ||
466 To == std::next(OrigDAGInterval.bottom()->getIterator())) {
471 getMemDGNodeBefore(InsertAfterN,
true, MemN));
476 getMemDGNodeBefore(BeforeToN,
false, MemN));
478 getMemDGNodeAfter(BeforeToN,
true, MemN));
482void DependencyGraph::notifyEraseInstr(Instruction *
I) {
488 auto *PrevMemN = getMemDGNodeBefore(MemN,
false);
489 auto *NextMemN = getMemDGNodeAfter(MemN,
false);
490 if (PrevMemN !=
nullptr)
491 PrevMemN->NextMemN = NextMemN;
492 if (NextMemN !=
nullptr)
493 NextMemN->PrevMemN = PrevMemN;
496 InstrToNodeMap.erase(
I);
507 auto NewInterval = Union.getSingleDiff(DAGInterval);
508 if (NewInterval.empty())
511 createNewNodes(NewInterval);
527 if (!DstRange.empty()) {
530 scanAndAddDeps(DstN, SrcRange);
535 if (MemDAGInterval.empty()) {
536 FullScan(NewInterval);
553 else if (DAGInterval.bottom()->comesBefore(NewInterval.top())) {
555 auto SrcRangeFull = MemDAGInterval.getUnionInterval(DstRange);
559 scanAndAddDeps(DstN, SrcRange);
563 else if (NewInterval.bottom()->comesBefore(DAGInterval.top())) {
577 FullScan(NewInterval);
593 auto DstRangeOld = MemDAGInterval;
596 scanAndAddDeps(DstN, SrcRange);
609 Nodes.
reserve(InstrToNodeMap.size());
610 for (
const auto &Pair : InstrToNodeMap)
616 for (
auto *
N : Nodes)
std::pair< uint64_t, uint64_t > Interval
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
bool mayWriteToMemory() const LLVM_READONLY
Return true if this instruction may modify memory.
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class implements an extremely fast bulk output stream that can only output to a stream.
A DependencyGraph Node that points to an Instruction and contains memory dependency edges.
virtual void print(raw_ostream &OS, bool PrintDeps=true) const
static bool isMemDepCandidate(Instruction *I)
We consider I as a Memory Dependency Candidate instruction if it reads/write memory or if it has side...
unsigned UnscheduledSuccs
The number of unscheduled successors.
SchedBundle * SB
The scheduler bundle that this node belongs to.
bool Scheduled
This is true if this node has been scheduled.
static bool isMemDepNodeCandidate(Instruction *I)
\Returns true if I is a memory dependency candidate instruction.
static bool isFenceLike(Instruction *I)
\Returns true if I is fence like. It excludes non-mem intrinsics.
LLVM_DUMP_METHOD void dump() const
Instruction * getInstruction() const
static bool isStackSaveOrRestoreIntrinsic(Instruction *I)
LLVM_DUMP_METHOD void dump() const
DGNode * getNode(Instruction *I) const
DGNode * getNodeOrNull(Instruction *I) const
Like getNode() but returns nullptr if I is nullptr.
void print(raw_ostream &OS) const
DGNode * getOrCreateNode(Instruction *I)
Interval< Instruction > extend(ArrayRef< Instruction * > Instrs)
Build/extend the dependency graph such that it includes Instrs.
A sandboxir::User with operands, opcode and linked with previous/next instructions in an instruction ...
bool mayWriteToMemory() const
bool mayReadFromMemory() const
bool isTerminator() const
static MemDGNode * getBotMemDGNode(const Interval< Instruction > &Intvl, const DependencyGraph &DAG)
Scans the instruction chain in Intvl bottom-up, returning the bottom-most MemDGNode,...
static MemDGNode * getTopMemDGNode(const Interval< Instruction > &Intvl, const DependencyGraph &DAG)
Scans the instruction chain in Intvl top-down, returning the top-most MemDGNode, or nullptr.
static Interval< MemDGNode > make(const Interval< Instruction > &Instrs, DependencyGraph &DAG)
Given Instrs it finds their closest mem nodes in the interval and returns the corresponding mem range...
A DependencyGraph Node for instructions that may read/write memory, or have some ordering constraints...
virtual void print(raw_ostream &OS, bool PrintDeps=true) const override
Iterate over both def-use and mem dependencies.
PredIterator & operator++()
bool operator==(const PredIterator &Other) const
@ Reverting
‍Tracking changes
TrackerState getState() const
\Returns the current state of the tracker.
OperandUseIterator op_iterator
static ModRefInfo aliasAnalysisGetModRefInfo(BatchAAResults &BatchAA, const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Equivalent to BatchAA::getModRefInfo().
static std::optional< llvm::MemoryLocation > memoryLocationGetOrNone(const Instruction *I)
Equivalent to MemoryLocation::getOrNone(I).
A SandboxIR Value has users. This is the base class.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BasicBlock
Various leaf nodes.
static bool isOrdered(Instruction *I)
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
auto reverse(ContainerTy &&C)
bool isModSet(const ModRefInfo MRI)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
ModRefInfo
Flags indicating whether a memory access modifies or references memory.
@ ModRef
The access may reference and may modify the value stored in memory.
DWARFExpression::Operation Op
bool isRefSet(const ModRefInfo MRI)