14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
36 Out =
C->getAPIntValue().getSExtValue();
41 Out =
C->getValueAPF().bitcastToAPInt().getSExtValue();
55 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
75 bool fp16SrcZerosHighBits(
unsigned Opc)
const;
93 std::pair<SDValue, SDValue> foldFrameIndex(
SDValue N)
const;
95 bool isInlineImmediate(
const SDNode *
N)
const;
97 bool isInlineImmediate(
const APInt &Imm)
const {
101 bool isInlineImmediate(
const APFloat &Imm)
const {
105 bool isVGPRImm(
const SDNode *
N)
const;
106 bool isUniformLoad(
const SDNode *
N)
const;
107 bool isUniformBr(
const SDNode *
N)
const;
111 bool isUnneededShiftMask(
const SDNode *
N,
unsigned ShAmtBits)
const;
113 bool isBaseWithConstantOffset64(SDValue
Addr, SDValue &
LHS,
116 MachineSDNode *buildSMovImm64(SDLoc &
DL,
uint64_t Val, EVT VT)
const;
118 SDNode *glueCopyToOp(SDNode *
N, SDValue NewChain, SDValue Glue)
const;
119 SDNode *glueCopyToM0(SDNode *
N, SDValue Val)
const;
120 SDNode *glueCopyToM0LDSInit(SDNode *
N)
const;
122 const TargetRegisterClass *getOperandRegClass(SDNode *
N,
unsigned OpNo)
const;
123 virtual bool SelectADDRVTX_READ(SDValue
Addr, SDValue &
Base, SDValue &
Offset);
124 virtual bool SelectADDRIndirect(SDValue
Addr, SDValue &
Base, SDValue &
Offset);
125 bool isDSOffsetLegal(SDValue
Base,
unsigned Offset)
const;
126 bool isDSOffset2Legal(SDValue
Base,
unsigned Offset0,
unsigned Offset1,
127 unsigned Size)
const;
129 bool isFlatScratchBaseLegal(SDValue
Addr)
const;
130 bool isFlatScratchBaseLegalSV(SDValue
Addr)
const;
131 bool isFlatScratchBaseLegalSVImm(SDValue
Addr)
const;
132 bool isSOffsetLegalWithImmOffset(SDValue *SOffset,
bool Imm32Only,
133 bool IsBuffer, int64_t ImmOffset = 0)
const;
135 bool SelectDS1Addr1Offset(SDValue
Ptr, SDValue &
Base, SDValue &
Offset)
const;
136 bool SelectDS64Bit4ByteAligned(SDValue
Ptr, SDValue &
Base, SDValue &Offset0,
137 SDValue &Offset1)
const;
138 bool SelectDS128Bit8ByteAligned(SDValue
Ptr, SDValue &
Base, SDValue &Offset0,
139 SDValue &Offset1)
const;
140 bool SelectDSReadWrite2(SDValue
Ptr, SDValue &
Base, SDValue &Offset0,
141 SDValue &Offset1,
unsigned Size)
const;
142 bool SelectMUBUF(SDValue
Addr, SDValue &SRsrc, SDValue &VAddr,
143 SDValue &SOffset, SDValue &
Offset, SDValue &Offen,
144 SDValue &Idxen, SDValue &Addr64)
const;
145 bool SelectMUBUFAddr64(SDValue
Addr, SDValue &SRsrc, SDValue &VAddr,
146 SDValue &SOffset, SDValue &
Offset)
const;
147 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue
Addr, SDValue &RSrc,
148 SDValue &VAddr, SDValue &SOffset,
149 SDValue &ImmOffset)
const;
150 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue
Addr, SDValue &SRsrc,
151 SDValue &Soffset, SDValue &
Offset)
const;
153 bool SelectMUBUFOffset(SDValue
Addr, SDValue &SRsrc, SDValue &Soffset,
155 bool SelectBUFSOffset(SDValue
Addr, SDValue &SOffset)
const;
157 bool SelectFlatOffsetImpl(SDNode *
N, SDValue
Addr, SDValue &VAddr,
159 bool SelectFlatOffset(SDNode *
N, SDValue
Addr, SDValue &VAddr,
161 bool SelectGlobalOffset(SDNode *
N, SDValue
Addr, SDValue &VAddr,
163 bool SelectScratchOffset(SDNode *
N, SDValue
Addr, SDValue &VAddr,
165 bool SelectGlobalSAddr(SDNode *
N, SDValue
Addr, SDValue &SAddr,
166 SDValue &VOffset, SDValue &
Offset,
bool &ScaleOffset,
167 bool NeedIOffset =
true)
const;
168 bool SelectGlobalSAddr(SDNode *
N, SDValue
Addr, SDValue &SAddr,
169 SDValue &VOffset, SDValue &
Offset,
170 SDValue &CPol)
const;
171 bool SelectGlobalSAddrCPol(SDNode *
N, SDValue
Addr, SDValue &SAddr,
172 SDValue &VOffset, SDValue &
Offset,
173 SDValue &CPol)
const;
174 bool SelectGlobalSAddrCPolM0(SDNode *
N, SDValue
Addr, SDValue &SAddr,
175 SDValue &VOffset, SDValue &
Offset,
176 SDValue &CPol)
const;
177 bool SelectGlobalSAddrGLC(SDNode *
N, SDValue
Addr, SDValue &SAddr,
178 SDValue &VOffset, SDValue &
Offset,
179 SDValue &CPol)
const;
180 bool SelectGlobalSAddrNoIOffset(SDNode *
N, SDValue
Addr, SDValue &SAddr,
181 SDValue &VOffset, SDValue &CPol)
const;
182 bool SelectScratchSAddr(SDNode *
N, SDValue
Addr, SDValue &SAddr,
184 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
186 bool SelectScratchSVAddr(SDNode *
N, SDValue
Addr, SDValue &VAddr,
187 SDValue &SAddr, SDValue &
Offset,
188 SDValue &CPol)
const;
190 bool SelectSMRDOffset(SDNode *
N, SDValue ByteOffsetNode, SDValue *SOffset,
191 SDValue *
Offset,
bool Imm32Only =
false,
192 bool IsBuffer =
false,
bool HasSOffset =
false,
193 int64_t ImmOffset = 0,
194 bool *ScaleOffset =
nullptr)
const;
195 SDValue Expand32BitAddress(SDValue
Addr)
const;
196 bool SelectSMRDBaseOffset(SDNode *
N, SDValue
Addr, SDValue &SBase,
197 SDValue *SOffset, SDValue *
Offset,
198 bool Imm32Only =
false,
bool IsBuffer =
false,
199 bool HasSOffset =
false, int64_t ImmOffset = 0,
200 bool *ScaleOffset =
nullptr)
const;
201 bool SelectSMRD(SDNode *
N, SDValue
Addr, SDValue &SBase, SDValue *SOffset,
202 SDValue *
Offset,
bool Imm32Only =
false,
203 bool *ScaleOffset =
nullptr)
const;
204 bool SelectSMRDImm(SDValue
Addr, SDValue &SBase, SDValue &
Offset)
const;
205 bool SelectSMRDImm32(SDValue
Addr, SDValue &SBase, SDValue &
Offset)
const;
206 bool SelectScaleOffset(SDNode *
N, SDValue &
Offset,
bool IsSigned)
const;
207 bool SelectSMRDSgpr(SDNode *
N, SDValue
Addr, SDValue &SBase, SDValue &SOffset,
208 SDValue &CPol)
const;
209 bool SelectSMRDSgprImm(SDNode *
N, SDValue
Addr, SDValue &SBase,
210 SDValue &SOffset, SDValue &
Offset,
211 SDValue &CPol)
const;
212 bool SelectSMRDBufferImm(SDValue
N, SDValue &
Offset)
const;
213 bool SelectSMRDBufferImm32(SDValue
N, SDValue &
Offset)
const;
214 bool SelectSMRDBufferSgprImm(SDValue
N, SDValue &SOffset,
216 bool SelectSMRDPrefetchImm(SDValue
Addr, SDValue &SBase,
218 bool SelectMOVRELOffset(SDValue
Index, SDValue &
Base, SDValue &
Offset)
const;
220 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src,
unsigned &SrcMods,
221 bool IsCanonicalizing =
true,
222 bool AllowAbs =
true)
const;
223 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
224 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
225 SDValue &SrcMods)
const;
226 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
227 bool SelectVOP3NoMods(SDValue In, SDValue &Src)
const;
228 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
229 SDValue &Clamp, SDValue &Omod)
const;
230 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
231 SDValue &Clamp, SDValue &Omod)
const;
232 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
233 SDValue &Clamp, SDValue &Omod)
const;
235 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
237 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
238 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
240 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
241 SDValue &Omod)
const;
243 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
244 bool IsDOT =
false)
const;
245 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
247 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src)
const;
249 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
250 SDValue &SrcMods)
const;
251 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
252 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
253 SDValue &SrcMods)
const;
254 bool SelectWMMAVISrc(SDValue In, SDValue &Src)
const;
256 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
257 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
258 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
260 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
262 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
263 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
unsigned &Mods,
265 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
266 SDValue &SrcMods)
const;
267 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
268 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
269 SDValue &SrcMods)
const;
270 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
271 SDValue &SrcMods)
const;
273 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
276 SDValue getHi16Elt(SDValue In)
const;
278 SDValue getMaterializedScalarImm32(int64_t Val,
const SDLoc &
DL)
const;
280 void SelectADD_SUB_I64(SDNode *
N);
281 void SelectAddcSubb(SDNode *
N);
282 void SelectUADDO_USUBO(SDNode *
N);
283 void SelectDIV_SCALE(SDNode *
N);
284 void SelectMAD_64_32(SDNode *
N);
285 void SelectMUL_LOHI(SDNode *
N);
286 void SelectFMA_W_CHAIN(SDNode *
N);
287 void SelectFMUL_W_CHAIN(SDNode *
N);
288 SDNode *getBFE32(
bool IsSigned,
const SDLoc &
DL, SDValue Val,
uint32_t Offset,
290 void SelectS_BFEFromShifts(SDNode *
N);
291 void SelectS_BFE(SDNode *
N);
292 bool isCBranchSCC(
const SDNode *
N)
const;
293 void SelectBRCOND(SDNode *
N);
294 void SelectFMAD_FMA(SDNode *
N);
295 void SelectFP_EXTEND(SDNode *
N);
296 void SelectDSAppendConsume(SDNode *
N,
unsigned IntrID);
297 void SelectDSBvhStackIntrinsic(SDNode *
N,
unsigned IntrID);
298 void SelectDS_GWS(SDNode *
N,
unsigned IntrID);
299 void SelectInterpP1F16(SDNode *
N);
300 void SelectINTRINSIC_W_CHAIN(SDNode *
N);
301 void SelectINTRINSIC_WO_CHAIN(SDNode *
N);
302 void SelectINTRINSIC_VOID(SDNode *
N);
303 void SelectWAVE_ADDRESS(SDNode *
N);
304 void SelectSTACKRESTORE(SDNode *
N);
308#include "AMDGPUGenDAGISel.inc"
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMD GCN specific subclass of TargetSubtarget.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
AMDGPU specific code to select AMDGPU machine instructions for SelectionDAG operations.
void SelectBuildVector(SDNode *N, unsigned RegClassID)
bool runOnMachineFunction(MachineFunction &MF) override
void SelectVectorShuffle(SDNode *N)
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
AMDGPUDAGToDAGISel()=delete
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool matchLoadD16FromBuildVector(SDNode *N) const
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Class for arbitrary precision integers.
A container for analyses that lazily runs them and caches their results.
Represent the analysis usage information of a pass.
const SIInstrInfo * getInstrInfo() const override
A set of analyses that are preserved following a run of a transformation pass.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isInlineConstant(const APInt &Imm) const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
StringRef - Represent a constant reference to a string, i.e.
Primary interface to the complete machine description for the target machine.
@ C
The default llvm calling convention, compatible with C.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
This is an optimization pass for GlobalISel generic memory operations.
static bool getConstantValue(SDValue N, uint32_t &Out)
CodeGenOptLevel
Code generation optimization level.
static SDNode * packConstantV2I16(const SDNode *N, SelectionDAG &DAG)