19#if defined(LLVM_HAVE_TF_AOT_REGALLOCEVICTMODEL) || defined(LLVM_HAVE_TFLITE)
45#include <unordered_map>
49#define DEBUG_TYPE "ml-regalloc"
52#if defined(LLVM_HAVE_TF_AOT_REGALLOCEVICTMODEL)
53#include "RegAllocEvictModel.h"
60 "regalloc-evict-interactive-channel-base",
cl::Hidden,
62 "Base file path for the interactive mode. The incoming filename should "
63 "have the name <regalloc-evict-interactive-channel-base>.in, while the "
64 "outgoing name should be "
65 "<regalloc-evict-interactive-channel-base>.out"));
69 cl::desc(
"The maximum number of times a live range can be "
70 "evicted before preventing it from being evicted"),
74#ifdef LLVM_HAVE_TFLITE
80 cl::desc(
"Training log for the register allocator eviction model"));
84 cl::desc(
"The model being trained for register allocation eviction"));
87 "regalloc-enable-development-features",
cl::Hidden,
88 cl::desc(
"Whether or not to enable features under development for the ML "
112 return "Register Allocation Pass Scoring";
134 "Register Allocation Scoring Pass",
false,
false)
142static const int OpcodeValueCutoff = 17716;
166#define RA_EVICT_FEATURES_LIST(M) \
167 M(int64_t, mask, PerLiveRangeShape, \
168 "boolean values, 0 for unavailable candidates (i.e. if a position is 0, " \
170 "can't be evicted)") \
171 M(int64_t, is_free, PerLiveRangeShape, \
172 "boolean values, 1 if this phys reg is actually free (no interferences)") \
173 M(float, nr_urgent, PerLiveRangeShape, \
174 "number of 'urgent' intervals, normalized. Urgent are those that are OK " \
175 "to break cascades") \
176 M(float, nr_broken_hints, PerLiveRangeShape, \
177 "if this position were evicted, how many broken hints would there be") \
178 M(int64_t, is_hint, PerLiveRangeShape, \
179 "is this a preferred phys reg for the candidate") \
180 M(int64_t, is_local, PerLiveRangeShape, \
181 "is this live range local to a basic block") \
182 M(float, nr_rematerializable, PerLiveRangeShape, \
183 "nr rematerializable ranges") \
184 M(float, nr_defs_and_uses, PerLiveRangeShape, \
185 "bb freq - weighed nr defs and uses") \
186 M(float, weighed_reads_by_max, PerLiveRangeShape, \
187 "bb freq - weighed nr of reads, normalized") \
188 M(float, weighed_writes_by_max, PerLiveRangeShape, \
189 "bb feq - weighed nr of writes, normalized") \
190 M(float, weighed_read_writes_by_max, PerLiveRangeShape, \
191 "bb freq - weighed nr of uses that are both read and writes, normalized") \
192 M(float, weighed_indvars_by_max, PerLiveRangeShape, \
193 "bb freq - weighed nr of uses that are indvars, normalized") \
194 M(float, hint_weights_by_max, PerLiveRangeShape, \
195 "bb freq - weighed nr of uses that are hints, normalized") \
196 M(float, start_bb_freq_by_max, PerLiveRangeShape, \
197 "the freq in the start block, normalized") \
198 M(float, end_bb_freq_by_max, PerLiveRangeShape, \
199 "freq of end block, normalized") \
200 M(float, hottest_bb_freq_by_max, PerLiveRangeShape, \
201 "hottest BB freq, normalized") \
202 M(float, liverange_size, PerLiveRangeShape, \
203 "size (instr index diff) of the LR") \
204 M(float, use_def_density, PerLiveRangeShape, \
205 "the max weight, as computed by the manual heuristic") \
206 M(int64_t, max_stage, PerLiveRangeShape, \
207 "largest stage of an interval in this LR") \
208 M(int64_t, min_stage, PerLiveRangeShape, \
209 "lowest stage of an interval in this LR") \
210 M(float, progress, {1}, "ratio of current queue size to initial size")
212#ifdef LLVM_HAVE_TFLITE
213#define RA_EVICT_FIRST_DEVELOPMENT_FEATURE(M) \
214 M(int64_t, instructions, InstructionsShape, \
215 "Opcodes of the instructions covered by the eviction problem")
217#define RA_EVICT_REST_DEVELOPMENT_FEATURES(M) \
218 M(int64_t, instructions_mapping, InstructionsMappingShape, \
219 "A binary matrix mapping LRs to instruction opcodes") \
220 M(float, mbb_frequencies, MBBFrequencyShape, \
221 "A vector of machine basic block frequencies") \
222 M(int64_t, mbb_mapping, InstructionsShape, \
223 "A vector of indices mapping instructions to MBBs")
225#define RA_EVICT_FIRST_DEVELOPMENT_FEATURE(M)
226#define RA_EVICT_REST_DEVELOPMENT_FEATURES(M)
233#define DecisionName "index_to_evict"
239#define _FEATURE_IDX_SIMPLE(_, name, __, ___) name
240#define _FEATURE_IDX(A, B, C, D) _FEATURE_IDX_SIMPLE(A, B, C, D),
242#ifdef LLVM_HAVE_TFLITE
249#undef _FEATURE_IDX_SIMPLE
256template <
typename T>
size_t getTotalSize(
const std::vector<int64_t> &Shape) {
257 size_t Ret =
sizeof(
T);
258 for (
const auto V : Shape)
264#define _RESET(TYPE, NAME, SHAPE, __) \
265 std::memset(Runner.getTensorUntyped(FeatureIDs::NAME), 0, \
266 getTotalSize<TYPE>(SHAPE));
277struct LIFeatureComponents {
281 double IndVarUpdates = 0;
282 double HintWeights = 0.0;
283 int64_t NumDefsAndUses = 0;
284 float HottestBlockFreq = 0.0;
285 bool IsRemat =
false;
288using CandidateRegList =
290using FeaturesListNormalizer =
314 tryFindEvictionCandidatePosition(
const LiveInterval &VirtReg,
316 unsigned OrderLimit,
uint8_t CostPerUseLimit,
337 int64_t IsHint, int64_t LocalIntfsCount,
float NumUrgent,
345 return getDefaultAdvisor().canEvictHintInterference(VirtReg, PhysReg,
349 const LIFeatureComponents &
363 std::bitset<FeatureIDs::FeatureCount> DoNotNormalize;
364 const float InitialQSize;
369 mutable std::unordered_map<unsigned, unsigned> VirtRegEvictionCounts;
371 void onEviction(
Register RegBeingEvicted)
const {
375 ++VirtRegEvictionCounts[RegBeingEvicted.
id()];
379 auto EvictionCountIt = VirtRegEvictionCounts.
find(
Reg.
id());
380 if (EvictionCountIt != VirtRegEvictionCounts.end())
381 return EvictionCountIt->second;
386#define _DECL_FEATURES(type, name, shape, _) \
387 TensorSpec::createSpec<type>(#name, shape),
393class ReleaseModeEvictionAdvisorProvider final
396 ReleaseModeEvictionAdvisorProvider(
LLVMContext &Ctx)
411 std::unique_ptr<RegAllocEvictionAdvisor>
416 Runner = std::make_unique<ReleaseModeModelRunner<CompiledModelType>>(
419 Runner = std::make_unique<InteractiveModelRunner>(
425 "Invalid provider state: must have analysis available");
426 return std::make_unique<MLEvictAdvisor>(MF,
RA, Runner.get(), *MBFI,
432 std::unique_ptr<MLModelRunner> Runner;
435class ReleaseModeEvictionAdvisorAnalysisLegacy final
438 ReleaseModeEvictionAdvisorAnalysisLegacy()
448 std::make_unique<ReleaseModeEvictionAdvisorProvider>(M.getContext());
468#ifdef LLVM_HAVE_TFLITE
469static const TensorSpec Reward = TensorSpec::createSpec<float>(
"reward", {1});
475#define _DECL_TRAIN_FEATURES(type, name, shape, _) \
476 TensorSpec::createSpec<type>(std::string("action_") + #name, shape),
478class DevelopmentModeEvictAdvisor :
public MLEvictAdvisor {
484 : MLEvictAdvisor(MF,
RA, Runner, MBFI,
Loops), Log(Log) {}
487 int64_t tryFindEvictionCandidatePosition(
489 unsigned OrderLimit,
uint8_t CostPerUseLimit,
495class DevelopmentModeEvictionAdvisorProvider final
498 DevelopmentModeEvictionAdvisorProvider(
LLVMContext &Ctx)
504 TrainingInputFeatures = {
508 TensorSpec::createSpec<float>(
"action_discount", {1}),
509 TensorSpec::createSpec<int32_t>(
"action_step_type", {1}),
510 TensorSpec::createSpec<float>(
"action_reward", {1})};
513 TrainingInputFeatures = {
515 TensorSpec::createSpec<float>(
"action_discount", {1}),
516 TensorSpec::createSpec<int32_t>(
"action_step_type", {1}),
517 TensorSpec::createSpec<float>(
"action_reward", {1})};
519 if (ModelUnderTraining.empty() && TrainingLog.empty()) {
520 Ctx.
emitError(
"Regalloc development mode should be requested with at "
521 "least logging enabled and/or a training model");
524 if (ModelUnderTraining.empty())
525 Runner = std::make_unique<NoInferenceModelRunner>(Ctx,
InputFeatures);
527 Runner = ModelUnderTrainingRunner::createAndEnsureValid(
528 Ctx, ModelUnderTraining,
DecisionName, TrainingInputFeatures);
530 Ctx.
emitError(
"Regalloc: could not set up the model runner");
533 if (TrainingLog.empty())
536 auto OS = std::make_unique<raw_fd_ostream>(TrainingLog, EC);
538 Ctx.
emitError(EC.message() +
":" + TrainingLog);
542 if (
auto *MUTR = dyn_cast<ModelUnderTrainingRunner>(Runner.get()))
549 Log = std::make_unique<Logger>(std::move(
OS), LFS, Reward,
561 if (!Log || !Log->hasAnyObservationForContext(MF.
getName()))
567 if (Log->currentContext() != MF.
getName()) {
569 "The training log context shouldn't have had changed.");
571 if (Log->hasObservationInProgress())
572 Log->logReward<
float>(GetReward());
575 std::unique_ptr<RegAllocEvictionAdvisor>
581 Log->switchContext(MF.
getName());
583 "Invalid provider state: must have analysis available");
584 return std::make_unique<DevelopmentModeEvictAdvisor>(
585 MF,
RA, Runner.get(), *MBFI, *
Loops, Log.get());
590 std::vector<TensorSpec> TrainingInputFeatures;
592 std::unique_ptr<MLModelRunner> Runner;
593 std::unique_ptr<Logger> Log;
596class DevelopmentModeEvictionAdvisorAnalysisLegacy final
599 DevelopmentModeEvictionAdvisorAnalysisLegacy()
603 Provider = std::make_unique<DevelopmentModeEvictionAdvisorProvider>(
610 Provider->logRewardIfNeeded(MF, GetReward);
630 unsigned NumUsedRegs = 0;
631 for (
unsigned I = 0, E =
MRI.getNumVirtRegs();
I != E; ++
I) {
633 if (!
MRI.reg_nodbg_empty(Reg))
636 return static_cast<float>(NumUsedRegs);
645 InitialQSize(MLEvictAdvisor::getInitialQueueSize(MF)) {
648 DoNotNormalize.set(FeatureIDs::mask);
649 DoNotNormalize.set(FeatureIDs::is_free);
650 DoNotNormalize.set(FeatureIDs::is_hint);
651 DoNotNormalize.set(FeatureIDs::is_local);
652 DoNotNormalize.set(FeatureIDs::min_stage);
653 DoNotNormalize.set(FeatureIDs::max_stage);
654 DoNotNormalize.set(FeatureIDs::progress);
657int64_t MLEvictAdvisor::tryFindEvictionCandidatePosition(
666bool MLEvictAdvisor::loadInterferenceFeatures(
677 const bool IsLocal = LIS->intervalIsInOneMBB(VirtReg);
678 int64_t LocalIntfs = 0;
679 float NumUrgent = 0.0f;
682 unsigned Cascade =
RA.getExtraInfo().getCascadeOrCurrentNext(VirtReg.
reg());
690 if (IFIntervals.empty() && InterferingIntervals.
empty())
694 InterferingIntervals.
append(IFIntervals.begin(), IFIntervals.end());
696 assert(Intf->reg().isVirtual() &&
697 "Only expecting virtual register interference from query");
704 if (FixedRegisters.
count(Intf->reg()))
706 if (
RA.getExtraInfo().getStage(*Intf) ==
RS_Done)
710 (Intf->isSpillable() ||
711 RegClassInfo.getNumAllocatableRegs(
MRI->getRegClass(VirtReg.
reg())) <
712 RegClassInfo.getNumAllocatableRegs(
713 MRI->getRegClass(Intf->reg())));
715 unsigned IntfCascade =
RA.getExtraInfo().getCascade(Intf->reg());
726 if (Cascade <= IntfCascade) {
732 LocalIntfs += (IsLocal && LIS->intervalIsInOneMBB(*Intf) &&
733 (!EnableLocalReassign || !canReassign(*Intf, PhysReg)));
738 extractFeatures(InterferingIntervals, Largest, Pos, IsHint, LocalIntfs,
739 NumUrgent, LRPosInfo);
743MCRegister MLEvictAdvisor::tryFindEvictionCandidate(
746 auto MaybeOrderLimit = getOrderLimit(VirtReg, Order, CostPerUseLimit);
747 if (!MaybeOrderLimit)
749 unsigned OrderLimit = *MaybeOrderLimit;
757 const bool MustFindEviction =
763 resetInputs(*Runner);
768 CandidateRegList Regs;
769 Regs.fill({0,
false});
775 FeaturesListNormalizer Largest(FeatureIDs::FeatureCount, 0.0);
787 assert(!Regs[Pos].second);
789 if (!canAllocatePhysReg(CostPerUseLimit, PhysReg)) {
792 if (loadInterferenceFeatures(VirtReg, PhysReg,
I.isHint(), FixedRegisters,
793 Largest, Pos, LRPosInfo)) {
795 Regs[Pos] = std::make_pair(PhysReg,
true);
800 assert(!MustFindEviction);
803 const size_t ValidPosLimit = Pos;
807 if (!MustFindEviction)
812 assert(InitialQSize > 0.0 &&
"We couldn't have gotten here if we had "
813 "nothing to allocate initially.");
814#ifdef LLVM_HAVE_TFLITE
819 auto *CurrentMachineInstruction =
820 LIS->getInstructionFromIndex(InputIndex);
821 if (!CurrentMachineInstruction) {
824 return CurrentMachineInstruction->getOpcode();
827 auto *CurrentMachineInstruction =
828 LIS->getInstructionFromIndex(InputIndex);
830 CurrentMachineInstruction->getParent());
833 auto *CurrentMachineInstruction =
834 LIS->getInstructionFromIndex(InputIndex);
835 return CurrentMachineInstruction->
getParent();
837 FeatureIDs::instructions, FeatureIDs::instructions_mapping,
838 FeatureIDs::mbb_frequencies, FeatureIDs::mbb_mapping,
839 LIS->getSlotIndexes()->getLastIndex());
843 for (
auto &V : Largest)
853 *Runner->
getTensor<
float>(FeatureIDs::progress) =
854 static_cast<float>(
RA.getQueueSize()) / InitialQSize;
857 size_t CandidatePos = tryFindEvictionCandidatePosition(
858 VirtReg, Order, OrderLimit, CostPerUseLimit, FixedRegisters);
861 assert(Regs[CandidatePos].second);
863 onEviction(VirtReg.
reg());
864 assert(!MustFindEviction);
867 assert(CandidatePos < ValidPosLimit);
873 for (
MCRegUnit Unit :
TRI->regunits(Regs[CandidatePos].first)) {
877 onEviction(Intf->reg());
881 return Regs[CandidatePos].first;
884const LIFeatureComponents &
885MLEvictAdvisor::getLIFeatureComponents(
const LiveInterval &LI)
const {
887 LIFeatureComponents
Empty;
888 auto I = CachedFeatures.insert(std::make_pair(
ID, Empty));
889 LIFeatureComponents &
Ret =
I.first->getSecond();
897 I =
MRI->reg_instr_nodbg_begin(LI.
reg()),
898 E =
MRI->reg_instr_nodbg_end();
902 ++
Ret.NumDefsAndUses;
906 if (
MI->isIdentityCopy() ||
MI->isImplicitDef())
910 std::tie(Reads,
Writes) =
MI->readsWritesVirtualRegister(LI.
reg());
913 Ret.HottestBlockFreq = std::max(Freq,
Ret.HottestBlockFreq);
919 auto *
MBB =
MI->getParent();
923 if (
Writes && IsExiting && LIS->isLiveOutOfMBB(LI,
MBB))
924 Ret.IndVarUpdates += Freq;
927 Ret.HintWeights += Freq;
936void MLEvictAdvisor::extractFeatures(
939 int64_t LocalIntfsCount,
float NumUrgent,
941 int64_t NumDefsAndUses = 0;
942 int64_t NumBrokenHints = 0;
946 double IndVarUpdates = 0.0;
947 double HintWeights = 0.0;
948 float StartBBFreq = 0.0;
949 float EndBBFreq = 0.0;
950 float HottestBlockFreq = 0.0;
951 int32_t NumRematerializable = 0;
952 float TotalWeight = 0.0;
954 SlotIndex EndSI = LIS->getSlotIndexes()->getZeroIndex();
955 SlotIndex StartSI = LIS->getSlotIndexes()->getLastIndex();
956 int64_t MaxStage = 0;
958 Intervals.
empty() ? 0 : std::numeric_limits<int64_t>::max();
960 for (
const auto *L : Intervals) {
962 MaxStage = std::max<int64_t>(
963 MaxStage,
static_cast<int64_t
>(
RA.getExtraInfo().getStage(LI)));
964 MinStage = std::min<int64_t>(
965 MinStage,
static_cast<int64_t
>(
RA.getExtraInfo().getStage(LI)));
967 TotalWeight = std::max(TotalWeight, LI.
weight());
974 const LIFeatureComponents &LIFC = getLIFeatureComponents(LI);
975 NumBrokenHints += VRM->hasPreferredPhys(LI.
reg());
977 NumDefsAndUses += LIFC.NumDefsAndUses;
978 HottestBlockFreq = std::max(HottestBlockFreq, LIFC.HottestBlockFreq);
983 IndVarUpdates += LIFC.IndVarUpdates;
985 HintWeights += LIFC.HintWeights;
986 NumRematerializable += LIFC.IsRemat;
989 for (
auto CurrentSegment : LI) {
996 if (!Intervals.empty()) {
999 if (EndSI >= LIS->getSlotIndexes()->getLastIndex())
1000 EndSI = LIS->getSlotIndexes()->getLastIndex().
getPrevIndex();
1006#define SET(ID, TYPE, VAL) \
1008 Runner->getTensor<TYPE>(FeatureIDs::ID)[Pos] = static_cast<TYPE>(VAL); \
1009 if (!DoNotNormalize.test(FeatureIDs::ID)) \
1010 Largest[FeatureIDs::ID] = \
1011 std::max(Largest[FeatureIDs::ID], static_cast<float>(VAL)); \
1013 SET(mask, int64_t, 1);
1014 SET(is_free, int64_t, Intervals.empty());
1015 SET(nr_urgent,
float, NumUrgent);
1016 SET(nr_broken_hints,
float, NumBrokenHints);
1017 SET(is_hint, int64_t, IsHint);
1018 SET(is_local, int64_t, LocalIntfsCount);
1019 SET(nr_rematerializable,
float, NumRematerializable);
1020 SET(nr_defs_and_uses,
float, NumDefsAndUses);
1021 SET(weighed_reads_by_max,
float, R);
1022 SET(weighed_writes_by_max,
float, W);
1023 SET(weighed_read_writes_by_max,
float, RW);
1024 SET(weighed_indvars_by_max,
float, IndVarUpdates);
1025 SET(hint_weights_by_max,
float, HintWeights);
1026 SET(start_bb_freq_by_max,
float, StartBBFreq);
1027 SET(end_bb_freq_by_max,
float, EndBBFreq);
1028 SET(hottest_bb_freq_by_max,
float, HottestBlockFreq);
1029 SET(liverange_size,
float,
Size);
1030 SET(use_def_density,
float, TotalWeight);
1031 SET(max_stage, int64_t, MaxStage);
1032 SET(min_stage, int64_t, MinStage);
1041 const int InstructionsIndex,
const int InstructionsMappingIndex,
1042 const int MBBFreqIndex,
const int MBBMappingIndex,
1060 LRPosInfo.
begin(), LRPosInfo.
end(),
1062 size_t InstructionIndex = 0;
1063 size_t CurrentSegmentIndex = 0;
1064 SlotIndex CurrentIndex = LRPosInfo[0].Begin;
1065 std::map<MachineBasicBlock *, size_t> VisitedMBBs;
1066 size_t CurrentMBBIndex = 0;
1078 while (CurrentIndex <= LRPosInfo[CurrentSegmentIndex].
End &&
1080 int CurrentOpcode = GetOpcode(CurrentIndex);
1082 if (CurrentOpcode == -1) {
1085 if (CurrentIndex >= LastIndex) {
1092 if (VisitedMBBs.count(CurrentMBBReference) == 0) {
1093 VisitedMBBs[CurrentMBBReference] = CurrentMBBIndex;
1097 GetMBBFreq, CurrentMBBReference, RegallocRunner,
1098 MBBFreqIndex, MBBMappingIndex);
1100 assert(LRPosInfo[CurrentSegmentIndex].Begin <= CurrentIndex);
1101 RegallocRunner->
getTensor<int64_t>(InstructionsIndex)[InstructionIndex] =
1102 CurrentOpcode < OpcodeValueCutoff ? CurrentOpcode : 0;
1104 auto CurrentSegmentPosition = LRPosInfo[CurrentSegmentIndex].Pos;
1106 InstructionsMappingIndex)[CurrentSegmentPosition *
1108 InstructionIndex] = 1;
1117 size_t OverlapCheckCurrentSegment = CurrentSegmentIndex + 1;
1118 while (OverlapCheckCurrentSegment < LRPosInfo.
size() &&
1119 LRPosInfo[OverlapCheckCurrentSegment].Begin <= CurrentIndex) {
1120 auto OverlapCurrentSegmentPosition =
1121 LRPosInfo[OverlapCheckCurrentSegment].Pos;
1122 if (LRPosInfo[OverlapCheckCurrentSegment].
End >= CurrentIndex) {
1124 InstructionsMappingIndex)[OverlapCurrentSegmentPosition *
1126 InstructionIndex] = 1;
1128 ++OverlapCheckCurrentSegment;
1131 if (CurrentIndex >= LastIndex) {
1138 if (CurrentSegmentIndex == LRPosInfo.
size() - 1 ||
1145 if (LRPosInfo[CurrentSegmentIndex + 1].Begin >
1146 LRPosInfo[CurrentSegmentIndex].
End) {
1147 CurrentIndex = LRPosInfo[CurrentSegmentIndex + 1].Begin;
1149 ++CurrentSegmentIndex;
1154 const SlotIndex CurrentIndex,
const size_t CurrentInstructionIndex,
1155 std::map<MachineBasicBlock *, size_t> &VisitedMBBs,
1158 const int MBBFreqIndex,
const int MBBMappingIndex) {
1159 size_t CurrentMBBIndex = VisitedMBBs[CurrentMBBReference];
1160 float CurrentMBBFreq = GetMBBFreq(CurrentIndex);
1162 RegallocRunner->
getTensor<
float>(MBBFreqIndex)[CurrentMBBIndex] =
1165 MBBMappingIndex)[CurrentInstructionIndex] = CurrentMBBIndex;
1170#ifdef LLVM_HAVE_TFLITE
1174 return new DevelopmentModeEvictionAdvisorAnalysisLegacy();
1177int64_t DevelopmentModeEvictAdvisor::tryFindEvictionCandidatePosition(
1179 unsigned OrderLimit,
uint8_t CostPerUseLimit,
1182 if (isa<ModelUnderTrainingRunner>(getRunner())) {
1183 Ret = MLEvictAdvisor::tryFindEvictionCandidatePosition(
1184 VirtReg, Order, OrderLimit, CostPerUseLimit, FixedRegisters);
1186 MCRegister PhysReg = getDefaultAdvisor().tryFindEvictionCandidate(
1187 VirtReg, Order, CostPerUseLimit, FixedRegisters);
1199 if (TrainingLog.empty())
1204 if (Log->hasObservationInProgress())
1205 Log->logReward<
float>(0.0);
1207 Log->startObservation();
1208 size_t CurrentFeature = 0;
1210 ? FeatureIDs::FeaturesWithDevelopmentCount
1211 : FeatureIDs::FeatureCount;
1212 for (; CurrentFeature <
FeatureCount; ++CurrentFeature) {
1213 Log->logTensorValue(CurrentFeature,
1214 reinterpret_cast<const char *
>(
1215 getRunner().getTensorUntyped(CurrentFeature)));
1217 if (
auto *MUTR = dyn_cast<ModelUnderTrainingRunner>(&getRunner()))
1218 for (
size_t I = 0;
I < MUTR->extraOutputsForLoggingSpecs().size();
1219 ++
I, ++CurrentFeature)
1220 Log->logTensorValue(
1222 reinterpret_cast<const char *
>(MUTR->getUntypedExtraOutputValue(
I)));
1224 Log->logTensorValue(CurrentFeature,
reinterpret_cast<const char *
>(&Ret));
1225 Log->endObservation();
1230 std::optional<float> CachedReward;
1231 auto GetReward = [&]() {
1233 CachedReward =
static_cast<float>(
1235 MF, getAnalysis<MachineBlockFrequencyInfoWrapperPass>().getMBFI())
1237 return *CachedReward;
1240 getAnalysis<RegAllocEvictionAdvisorAnalysisLegacy>().logRewardIfNeeded(
1242 getAnalysis<RegAllocPriorityAdvisorAnalysisLegacy>().logRewardIfNeeded(
1250 return new ReleaseModeEvictionAdvisorProvider(Ctx);
1255#if defined(LLVM_HAVE_TFLITE)
1256 return new DevelopmentModeEvictionAdvisorProvider(Ctx);
1263 return llvm::isEmbeddedModelEvaluatorValid<CompiledModelType>() ||
1265 ?
new ReleaseModeEvictionAdvisorAnalysisLegacy()
1270#if !defined(LLVM_HAVE_TFLITE)
unsigned const MachineRegisterInfo * MRI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
SmallVector< uint32_t, 0 > Writes
@ Available
We know the block is fully available. This is a fixpoint.
Module.h This file contains the declarations for the Module class.
static cl::opt< std::string > InteractiveChannelBaseName("inliner-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <inliner-interactive-channel-base>.in, while the " "outgoing name should be <inliner-interactive-channel-base>.out"))
static const bool EnableDevelopmentFeatures
static cl::opt< unsigned > MaxEvictionCount("mlregalloc-max-eviction-count", cl::Hidden, cl::desc("The maximum number of times a live range can be " "evicted before preventing it from being evicted"), cl::init(100))
#define RA_EVICT_FEATURES_LIST(M)
#define _FEATURE_IDX_SIMPLE(_, name, __, ___)
#define RA_EVICT_FIRST_DEVELOPMENT_FEATURE(M)
#define SET(ID, TYPE, VAL)
#define _RESET(TYPE, NAME, SHAPE, __)
#define RA_EVICT_REST_DEVELOPMENT_FEATURES(M)
static cl::opt< std::string > InteractiveChannelBaseName("regalloc-evict-interactive-channel-base", cl::Hidden, cl::desc("Base file path for the interactive mode. The incoming filename should " "have the name <regalloc-evict-interactive-channel-base>.in, while the " "outgoing name should be " "<regalloc-evict-interactive-channel-base>.out"))
#define _FEATURE_IDX(A, B, C, D)
#define _DECL_FEATURES(type, name, shape, _)
Register const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
SI optimize exec mask operations pre RA
Iterator getOrderLimitEnd(unsigned OrderLimit) const
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
iterator find(const_arg_type_t< KeyT > Val)
FunctionPass class - This class is used to implement most global optimizations.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Query interferences between a single live virtual register and a live interval union.
const SmallVectorImpl< const LiveInterval * > & interferingVRegs(unsigned MaxInterferingRegs=std::numeric_limits< unsigned >::max())
LiveInterval - This class represents the liveness of a register, or stack slot.
bool isSpillable() const
isSpillable - Can this interval be spilled?
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
SlotIndex endIndex() const
endNumber - return the maximum point of the range of the whole, exclusive.
@ IK_VirtReg
Virtual register interference.
Logging utility - given an ordered specification of features, and assuming a scalar reward,...
bool isLoopExiting(const BlockT *BB) const
True if terminator in the block can branch to another block that is outside of the current loop.
Represents a single loop in the control flow graph.
Wrapper class representing physical registers. Should be passed by value.
static constexpr unsigned NoRegister
MLModelRunner interface: abstraction of a mechanism for evaluating a ML model.
virtual void switchContext(StringRef Name)
T * getTensor(I FeatureID)
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
double getBlockFreqRelativeToEntryBlock(const MachineBasicBlock *MBB) const
Compute the frequency of the block, relative to the entry block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Representation of each machine instruction.
defusechain_iterator - This class provides iterator support for machine operands in the function that...
A Module instance is used to store all the information related to an LLVM module.
A mock class satisfying the interface expected by ReleaseModeModelRunner for its TGen parameter.
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual bool doInitialization(Module &)
doInitialization - Virtual method overridden by subclasses to do any necessary initialization before ...
ImmutableAnalysis abstraction for fetching the Eviction Advisor.
virtual void logRewardIfNeeded(const MachineFunction &MF, function_ref< float()> GetReward)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
Common provider for legacy and new pass managers.
virtual std::unique_ptr< RegAllocEvictionAdvisor > getAdvisor(const MachineFunction &MF, const RAGreedy &RA, MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops)=0
virtual void logRewardIfNeeded(const MachineFunction &MF, llvm::function_ref< float()> GetReward)
virtual bool canEvictHintInterference(const LiveInterval &VirtReg, MCRegister PhysReg, const SmallVirtRegSet &FixedRegisters) const =0
Find out if we can evict the live ranges occupying the given PhysReg, which is a hint (preferred regi...
virtual MCRegister tryFindEvictionCandidate(const LiveInterval &VirtReg, const AllocationOrder &Order, uint8_t CostPerUseLimit, const SmallVirtRegSet &FixedRegisters) const =0
Find a physical register that can be freed by evicting the FixedRegisters, or return NoRegister.
LLVM_ABI_FOR_TEST double getScore() const
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
void getAnalysisUsage(AnalysisUsage &AU) const override
RegAllocReward analysis usage.
~RegAllocScoring() override=default
bool runOnMachineFunction(MachineFunction &) override
Performs this pass.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
constexpr unsigned id() const
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getNextIndex() const
Returns the next index.
int distance(SlotIndex other) const
Return the distance from this index to the given one.
SlotIndex getPrevIndex() const
Returns the previous index.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
static bool isRematerializable(const LiveInterval &LI, const LiveIntervals &LIS, const VirtRegMap &VRM, const TargetInstrInfo &TII)
Determine if all values in LI are rematerializable.
static Register copyHint(const MachineInstr *MI, Register Reg, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI)
Return the preferred allocation register for reg, given a COPY instruction.
An efficient, type-erasing, non-owning reference to a callable.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
static const int ModelMaxSupportedInstructionCount
RegAllocEvictionAdvisorAnalysisLegacy * createReleaseModeAdvisorAnalysisLegacy()
static const int64_t ModelMaxSupportedMBBCount
RegAllocEvictionAdvisorProvider * createDevelopmentModeAdvisorProvider(LLVMContext &Ctx)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
LLVM_ABI_FOR_TEST void extractInstructionFeatures(llvm::SmallVectorImpl< LRStartEndInfo > &LRPosInfo, MLModelRunner *RegallocRunner, function_ref< int(SlotIndex)> GetOpcode, function_ref< float(SlotIndex)> GetMBBFreq, function_ref< MachineBasicBlock *(SlotIndex)> GetMBBReference, const int InstructionsIndex, const int InstructionsMappingIndex, const int MBBFreqIndex, const int MBBMappingIndex, const SlotIndex LastIndex)
static const TensorSpec DecisionSpec
RegAllocScore calculateRegAllocScore(const MachineFunction &MF, const MachineBlockFrequencyInfo &MBFI)
Calculate a score.
auto reverse(ContainerTy &&C)
LLVM_ABI const char *const DecisionName
LLVM_ABI void initializeRegAllocScoringPass(PassRegistry &)
static const std::vector< TensorSpec > InputFeatures
@ RS_Done
There is nothing more we can do to this live range.
LLVM_ABI FunctionPass * createRegAllocScoringPass()
When learning an eviction policy, extract score(reward) information, otherwise this does nothing.
cl::opt< unsigned > EvictInterferenceCutoff
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
LLVM_ATTRIBUTE_RETURNS_NONNULL RegAllocEvictionAdvisorProvider * createReleaseModeAdvisorProvider(LLVMContext &Ctx)
LLVM_ABI_FOR_TEST void extractMBBFrequency(const SlotIndex CurrentIndex, const size_t CurrentInstructionIndex, std::map< MachineBasicBlock *, size_t > &VisitedMBBs, function_ref< float(SlotIndex)> GetMBBFreq, MachineBasicBlock *CurrentMBBReference, MLModelRunner *RegallocRunner, const int MBBFreqIndex, const int MBBMappingIndex)
static const int64_t NumberOfInterferences
static const std::vector< int64_t > PerLiveRangeShape
RegAllocEvictionAdvisorAnalysisLegacy * createDevelopmentModeAdvisorAnalysisLegacy()
static const int64_t CandidateVirtRegPos
Implement std::hash so that hash_code can be used in STL containers.