LLVM 22.0.0git
AMDGPUISelDAGToDAG.h
Go to the documentation of this file.
1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16
17#include "GCNSubtarget.h"
24
25namespace llvm {
26
27static inline bool getConstantValue(SDValue N, uint32_t &Out) {
28 // This is only used for packed vectors, where using 0 for undef should
29 // always be good.
30 if (N.isUndef()) {
31 Out = 0;
32 return true;
33 }
34
35 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
36 Out = C->getAPIntValue().getSExtValue();
37 return true;
38 }
39
40 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N)) {
41 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
42 return true;
43 }
44
45 return false;
46}
47
48// TODO: Handle undef as zero
49static inline SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) {
50 assert(N->getOpcode() == ISD::BUILD_VECTOR && N->getNumOperands() == 2);
51 uint32_t LHSVal, RHSVal;
52 if (getConstantValue(N->getOperand(0), LHSVal) &&
53 getConstantValue(N->getOperand(1), RHSVal)) {
54 SDLoc SL(N);
55 uint32_t K = (LHSVal & 0xffff) | (RHSVal << 16);
56 return DAG.getMachineNode(AMDGPU::S_MOV_B32, SL, N->getValueType(0),
57 DAG.getTargetConstant(K, SL, MVT::i32));
58 }
59
60 return nullptr;
61}
62
63/// AMDGPU specific code to select AMDGPU machine instructions for
64/// SelectionDAG operations.
66 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
67 // make the right decision when generating code for different targets.
68 const GCNSubtarget *Subtarget;
69
70 // Default FP mode for the current function.
72
73 // Instructions that will be lowered with a final instruction that zeros the
74 // high result bits.
75 bool fp16SrcZerosHighBits(unsigned Opc) const;
76
77public:
79
81
84 void PreprocessISelDAG() override;
85 void Select(SDNode *N) override;
86 void PostprocessISelDAG() override;
87
88protected:
89 void SelectBuildVector(SDNode *N, unsigned RegClassID);
91
92private:
93 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
94
95 bool isInlineImmediate(const SDNode *N) const;
96
97 bool isInlineImmediate(const APInt &Imm) const {
98 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
99 }
100
101 bool isInlineImmediate(const APFloat &Imm) const {
102 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
103 }
104
105 bool isVGPRImm(const SDNode *N) const;
106 bool isUniformLoad(const SDNode *N) const;
107 bool isUniformBr(const SDNode *N) const;
108
109 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
110 // `ShAmtBits` bits is unneeded.
111 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
112
113 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
114 SDValue &RHS) const;
115
116 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
117
118 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
119 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
120 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
121
122 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
123 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
124 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
125 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
126 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
127 unsigned Size) const;
128
129 bool isFlatScratchBaseLegal(SDValue Addr) const;
130 bool isFlatScratchBaseLegalSV(SDValue Addr) const;
131 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
132 bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,
133 bool IsBuffer, int64_t ImmOffset = 0) const;
134
135 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
136 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
137 SDValue &Offset1) const;
138 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
139 SDValue &Offset1) const;
140 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
141 SDValue &Offset1, unsigned Size) const;
142 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
143 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
144 SDValue &Idxen, SDValue &Addr64) const;
145 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
146 SDValue &SOffset, SDValue &Offset) const;
147 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
148 SDValue &VAddr, SDValue &SOffset,
149 SDValue &ImmOffset) const;
150 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
151 SDValue &Soffset, SDValue &Offset) const;
152
153 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
154 SDValue &Offset) const;
155 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
156
157 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
158 SDValue &Offset, uint64_t FlatVariant) const;
159 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
160 SDValue &Offset) const;
161 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
162 SDValue &Offset) const;
163 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
164 SDValue &Offset) const;
165 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
166 SDValue &VOffset, SDValue &Offset, bool &ScaleOffset,
167 bool NeedIOffset = true) const;
168 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
169 SDValue &VOffset, SDValue &Offset,
170 SDValue &CPol) const;
171 bool SelectGlobalSAddrCPol(SDNode *N, SDValue Addr, SDValue &SAddr,
172 SDValue &VOffset, SDValue &Offset,
173 SDValue &CPol) const;
174 bool SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr, SDValue &SAddr,
175 SDValue &VOffset, SDValue &Offset,
176 SDValue &CPol) const;
177 bool SelectGlobalSAddrGLC(SDNode *N, SDValue Addr, SDValue &SAddr,
178 SDValue &VOffset, SDValue &Offset,
179 SDValue &CPol) const;
180 bool SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr, SDValue &SAddr,
181 SDValue &VOffset, SDValue &CPol) const;
182 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
183 SDValue &Offset) const;
184 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
185 uint64_t ImmOffset) const;
186 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
187 SDValue &SAddr, SDValue &Offset,
188 SDValue &CPol) const;
189
190 bool SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode, SDValue *SOffset,
191 SDValue *Offset, bool Imm32Only = false,
192 bool IsBuffer = false, bool HasSOffset = false,
193 int64_t ImmOffset = 0,
194 bool *ScaleOffset = nullptr) const;
195 SDValue Expand32BitAddress(SDValue Addr) const;
196 bool SelectSMRDBaseOffset(SDNode *N, SDValue Addr, SDValue &SBase,
197 SDValue *SOffset, SDValue *Offset,
198 bool Imm32Only = false, bool IsBuffer = false,
199 bool HasSOffset = false, int64_t ImmOffset = 0,
200 bool *ScaleOffset = nullptr) const;
201 bool SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase, SDValue *SOffset,
202 SDValue *Offset, bool Imm32Only = false,
203 bool *ScaleOffset = nullptr) const;
204 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
205 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
206 bool SelectScaleOffset(SDNode *N, SDValue &Offset, bool IsSigned) const;
207 bool SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
208 SDValue &CPol) const;
209 bool SelectSMRDSgprImm(SDNode *N, SDValue Addr, SDValue &SBase,
210 SDValue &SOffset, SDValue &Offset,
211 SDValue &CPol) const;
212 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
213 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
214 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
215 SDValue &Offset) const;
216 bool SelectSMRDPrefetchImm(SDValue Addr, SDValue &SBase,
217 SDValue &Offset) const;
218 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
219
220 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
221 bool IsCanonicalizing = true,
222 bool AllowAbs = true) const;
223 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
224 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
225 SDValue &SrcMods) const;
226 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
227 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
228 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
229 SDValue &Clamp, SDValue &Omod) const;
230 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
231 SDValue &Clamp, SDValue &Omod) const;
232 bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
233 SDValue &Clamp, SDValue &Omod) const;
234
235 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
236 bool OpSel) const;
237 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
238 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
239
240 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
241 SDValue &Omod) const;
242
243 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
244 bool IsDOT = false) const;
245 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
246
247 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
248
249 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
250 SDValue &SrcMods) const;
251 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
252 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
253 SDValue &SrcMods) const;
254 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
255
256 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
257 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
258 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey) const;
259
260 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
261
262 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
263 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods,
264 MVT VT) const;
265 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
266 SDValue &SrcMods) const;
267 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
268 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
269 SDValue &SrcMods) const;
270 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
271 SDValue &SrcMods) const;
272
273 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
274 SDValue &Tbl) const;
275
276 SDValue getHi16Elt(SDValue In) const;
277
278 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
279
280 void SelectADD_SUB_I64(SDNode *N);
281 void SelectAddcSubb(SDNode *N);
282 void SelectUADDO_USUBO(SDNode *N);
283 void SelectDIV_SCALE(SDNode *N);
284 void SelectMAD_64_32(SDNode *N);
285 void SelectMUL_LOHI(SDNode *N);
286 void SelectFMA_W_CHAIN(SDNode *N);
287 void SelectFMUL_W_CHAIN(SDNode *N);
288 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
289 uint32_t Width);
290 void SelectS_BFEFromShifts(SDNode *N);
291 void SelectS_BFE(SDNode *N);
292 bool isCBranchSCC(const SDNode *N) const;
293 void SelectBRCOND(SDNode *N);
294 void SelectFMAD_FMA(SDNode *N);
295 void SelectFP_EXTEND(SDNode *N);
296 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
297 void SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID);
298 void SelectDS_GWS(SDNode *N, unsigned IntrID);
299 void SelectInterpP1F16(SDNode *N);
300 void SelectINTRINSIC_W_CHAIN(SDNode *N);
301 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
302 void SelectINTRINSIC_VOID(SDNode *N);
303 void SelectWAVE_ADDRESS(SDNode *N);
304 void SelectSTACKRESTORE(SDNode *N);
305
306protected:
307 // Include the pieces autogenerated from the target description.
308#include "AMDGPUGenDAGISel.inc"
309};
310
312public:
314
317};
318
320public:
321 static char ID;
322
324
325 bool runOnMachineFunction(MachineFunction &MF) override;
326 void getAnalysisUsage(AnalysisUsage &AU) const override;
327 StringRef getPassName() const override;
328};
329
330} // namespace llvm
331
332#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU address space definition.
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
uint64_t Addr
uint32_t Index
uint64_t Size
AMD GCN specific subclass of TargetSubtarget.
Value * RHS
Value * LHS
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
AMDGPU specific code to select AMDGPU machine instructions for SelectionDAG operations.
void SelectBuildVector(SDNode *N, unsigned RegClassID)
bool runOnMachineFunction(MachineFunction &MF) override
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool matchLoadD16FromBuildVector(SDNode *N) const
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Class for arbitrary precision integers.
Definition: APInt.h:78
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:255
Represent the analysis usage information of a pass.
const SIInstrInfo * getInstrInfo() const override
Definition: GCNSubtarget.h:309
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:112
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isInlineConstant(const APInt &Imm) const
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
MachineFunction * MF
CodeGenOptLevel OptLevel
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:229
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:707
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:55
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:83
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:543
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:477
static bool getConstantValue(SDValue N, uint32_t &Out)
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:82
static SDNode * packConstantV2I16(const SDNode *N, SelectionDAG &DAG)
#define N