#include "RISCVISelLowering.h"
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCV.h"
#include "RISCVConstantPoolValue.h"
#include "RISCVMachineFunctionInfo.h"
#include "RISCVRegisterInfo.h"
#include "RISCVSelectionDAGInfo.h"
#include "RISCVSubtarget.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
#include "RISCVGenAsmMatcher.inc"
#include "RISCVGenSearchableTables.inc"

Classes
struct	VIDSequence

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.

namespace	llvm::RISCVVIntrinsicsTable

Macros
#define	DEBUG_TYPE "riscv-lower"

#define	OP_CASE(NODE)

#define	VP_CASE(NODE)

#define	CC_VLS_CASE(ABI_VLEN) case CallingConv::RISCV_VLSCall_##ABI_VLEN:

#define	GET_REGISTER_MATCHER

#define	GET_RISCVVIntrinsicsTable_IMPL

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")

static void	translateSetCCForBranch (const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	getVLOperand (SDValue Op)

static bool	useRVVForFixedLengthVectorVT (MVT VT, const RISCVSubtarget &Subtarget)

static MVT	getContainerForFixedLengthVector (const TargetLowering &TLI, MVT VT, const RISCVSubtarget &Subtarget)

static MVT	getContainerForFixedLengthVector (SelectionDAG &DAG, MVT VT, const RISCVSubtarget &Subtarget)

static SDValue	convertToScalableVector (EVT VT, SDValue V, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	convertFromScalableVector (EVT VT, SDValue V, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static MVT	getMaskTypeFor (MVT VecVT)
	Return the type of the mask type suitable for masking the provided vector type.

static SDValue	getAllOnesMask (MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
	Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.

static std::pair< SDValue, SDValue >	getDefaultScalableVLOps (MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static std::pair< SDValue, SDValue >	getDefaultVLOps (uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static std::pair< SDValue, SDValue >	getDefaultVLOps (MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerINT_TO_FP (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFP_TO_INT_SAT (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFP_TO_INT (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static RISCVFPRndMode::RoundingMode	matchRoundingOp (unsigned Opc)

static SDValue	lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerVectorXRINT_XROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	getVSlidedown (SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED)

static SDValue	getVSlideup (SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED)

static std::optional< APInt >	getExactInteger (const APFloat &APF, uint32_t BitWidth)

static std::optional< VIDSequence >	isSimpleVIDSequence (SDValue Op, unsigned EltSizeInBits)

static SDValue	matchSplatAsGather (SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerBuildVectorViaVID (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerBuildVectorViaDominantValues (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large proportion of the elements.

static SDValue	lowerBuildVectorOfConstants (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static unsigned	getPACKOpcode (unsigned DestBW, const RISCVSubtarget &Subtarget)

static SDValue	lowerBuildVectorViaPacking (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Double the element size of the build vector to reduce the number of vslide1down in the build vector chain.

static SDValue	lowerBUILD_VECTOR (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	splatPartsI64WithVL (const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)

static SDValue	splatSplitI64WithVL (const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)

static SDValue	lowerScalarSplat (SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerScalarInsert (SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	foldConcatVector (SDValue V1, SDValue V2)
	If concat_vector(V1,V2) could be folded away to some existing vector source, return it.

static SDValue	getSingleShuffleSrc (MVT VT, SDValue V1, SDValue V2)

static bool	isInterleaveShuffle (ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
	Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguous elements from another vector into the odd elements.

static bool	isMaskedSlidePair (ArrayRef< int > Mask, std::array< std::pair< int, int >, 2 > &SrcInfo)
	Is this mask representing a masked combination of two slides?

static bool	isElementRotate (const std::array< std::pair< int, int >, 2 > &SrcInfo, unsigned NumElts)

static bool	isAlternating (const std::array< std::pair< int, int >, 2 > &SrcInfo, ArrayRef< int > Mask, unsigned Factor, bool RequiredPolarity)

static bool	isZipEven (const std::array< std::pair< int, int >, 2 > &SrcInfo, ArrayRef< int > Mask, unsigned &Factor)
	Given a shuffle which can be represented as a pair of two slides, see if it is a zipeven idiom.

static bool	isZipOdd (const std::array< std::pair< int, int >, 2 > &SrcInfo, ArrayRef< int > Mask, unsigned &Factor)
	Given a shuffle which can be represented as a pair of two slides, see if it is a zipodd idiom.

static SDValue	getDeinterleaveShiftAndTrunc (const SDLoc &DL, MVT VT, SDValue Src, unsigned Factor, unsigned Index, SelectionDAG &DAG)

static SDValue	lowerVECTOR_SHUFFLEAsVRGatherVX (ShuffleVectorSDNode *SVN, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
	Match a single source shuffle which is an identity except that some particular element is repeated.

static SDValue	lowerVECTOR_SHUFFLEAsVSlidedown (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)

static SDValue	lowerVECTOR_SHUFFLEAsVSlideup (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)

static SDValue	lowerVECTOR_SHUFFLEAsVSlide1 (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
	Match v(f)slide1up/down idioms.

static SDValue	lowerVZIP (unsigned Opc, SDValue Op0, SDValue Op1, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	getWideningSpread (SDValue V, unsigned Factor, unsigned Index, const SDLoc &DL, SelectionDAG &DAG)

static SDValue	getWideningInterleave (SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerBitreverseShuffle (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static bool	isLegalBitRotate (ArrayRef< int > Mask, EVT VT, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)

static SDValue	lowerVECTOR_SHUFFLEAsRotate (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerShuffleViaVRegSplitting (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static bool	isCompressMask (ArrayRef< int > Mask)

static SDValue	lowerDisjointIndicesShuffle (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Given a shuffle where the indices are disjoint between the two sources, e.g.:

static bool	isLocalRepeatingShuffle (ArrayRef< int > Mask, int Span)
	Is this mask local (i.e.

static bool	isLowSourceShuffle (ArrayRef< int > Mask, int Span)
	Is this mask only using elements from the first span of the input?

static bool	isSpanSplatShuffle (ArrayRef< int > Mask, int Span)
	Return true for a mask which performs an arbitrary shuffle within the first span, and then repeats that same result across all remaining spans.

static SDValue	tryWidenMaskForShuffle (SDValue Op, SelectionDAG &DAG)
	Try to widen element type to get a new mask value for a better permutation sequence.

static SDValue	lowerVECTOR_SHUFFLE (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerConstant (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	LowerPREFETCH (SDValue Op, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)

static SDValue	LowerATOMIC_FENCE (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFMAXIMUM_FMINIMUM (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFABSorFNEG (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static unsigned	getRISCVVLOp (SDValue Op)
	Get a RISC-V target specified VL op for a given SDNode.

static bool	isPromotedOpNeedingSplit (SDValue Op, const RISCVSubtarget &Subtarget)

static SDValue	SplitVectorOp (SDValue Op, SelectionDAG &DAG)

static SDValue	SplitVPOp (SDValue Op, SelectionDAG &DAG)

static SDValue	SplitVectorReductionOp (SDValue Op, SelectionDAG &DAG)

static SDValue	SplitStrictFPVectorOp (SDValue Op, SelectionDAG &DAG)

static SDValue	getTargetNode (GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)

static SDValue	getTargetNode (BlockAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)

static SDValue	getTargetNode (ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)

static SDValue	getTargetNode (JumpTableSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)

static SDValue	getLargeGlobalAddress (GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)

static SDValue	getLargeExternalSymbol (ExternalSymbolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG)

static std::optional< bool >	matchSetCC (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)

static SDValue	combineSelectToBinOp (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	foldBinOpIntoSelectIfProfitable (SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static std::optional< MVT >	getSmallestVTForIndex (MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static bool	isValidVisniInsertExtractIndex (SDValue Idx)

static SDValue	lowerVectorIntrinsicScalars (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerGetVectorLength (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	lowerCttzElts (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static void	promoteVCIXScalar (const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)

static void	processVCIXOperands (SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)

static bool	isValidEGW (int EGS, EVT VT, const RISCVSubtarget &Subtarget)

static SDValue	getVCIXISDNodeWCHAIN (SDValue &Op, SelectionDAG &DAG, unsigned Type)

static SDValue	getVCIXISDNodeVOID (SDValue &Op, SelectionDAG &DAG, unsigned Type)

static SDValue	lowerFixedVectorSegLoadIntrinsics (unsigned IntNo, SDValue Op, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)

static SDValue	lowerFixedVectorSegStoreIntrinsics (unsigned IntNo, SDValue Op, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)

static unsigned	getRVVReductionOp (unsigned ISDOpcode)

static bool	isNonZeroAVL (SDValue AVL)

static SDValue	lowerReductionSeq (unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.

static std::tuple< unsigned, SDValue, SDValue >	getRVVFPReductionOpAndOperands (SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)

static SDValue	widenVectorOpsToi8 (SDValue N, const SDLoc &DL, SelectionDAG &DAG)

static unsigned	getRISCVWOpcode (unsigned Opcode)

static SDValue	customLegalizeToWOp (SDNode *N, SelectionDAG &DAG, unsigned ExtOpc=ISD::ANY_EXTEND)

static SDValue	customLegalizeToWOpWithSExt (SDNode *N, SelectionDAG &DAG)

static unsigned	getVecReduceOpcode (unsigned Opc)
	Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.

static SDValue	combineBinOpOfExtractToReduceTree (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed by scalar reduction as a vector reduction node.

static SDValue	combineBinOpToReduce (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	transformAddShlImm (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineShlAddIAddImpl (SDNode *N, SDValue AddI, SDValue Other, SelectionDAG &DAG)

static SDValue	combineShlAddIAdd (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineSelectAndUse (SDNode *N, SDValue Slct, SDValue OtherOp, SelectionDAG &DAG, bool AllOnes, const RISCVSubtarget &Subtarget)

static SDValue	combineSelectAndUseCommutative (SDNode *N, SelectionDAG &DAG, bool AllOnes, const RISCVSubtarget &Subtarget)

static SDValue	transformAddImmMulImm (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineBinOpOfZExt (SDNode *N, SelectionDAG &DAG)

static SDValue	combineAddOfBooleanXor (SDNode *N, SelectionDAG &DAG)

static SDValue	performADDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	combineSubOfBoolean (SDNode *N, SelectionDAG &DAG)

static SDValue	combineSubShiftToOrcB (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performSUBCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineDeMorganOfBoolean (SDNode *N, SelectionDAG &DAG)

static SDValue	combineTruncSelectToSMaxUSat (SDNode *N, SelectionDAG &DAG)

static SDValue	performTRUNCATECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	reverseZExtICmpCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	reduceANDOfAtomicLoad (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)

static SDValue	performANDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	combineOrOfCZERO (SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)

static SDValue	performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	performXORCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	expandMulToNAFSequence (SDNode *N, SelectionDAG &DAG, uint64_t MulAmt)

static SDValue	expandMulToAddOrSubOfShl (SDNode *N, SelectionDAG &DAG, uint64_t MulAmt)

static SDValue	expandMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	combineVectorMulToSraBitcast (SDNode *N, SelectionDAG &DAG)

static SDValue	performMULCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static bool	narrowIndex (SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
	According to the property that indexed load/store instructions zero-extend their indices, try to narrow the type of index operand.

static SDValue	combineVectorSizedSetCCEquality (EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Try to map an integer comparison with size > XLEN to vector instructions before type legalization splits it up into chunks.

static SDValue	performSETCCCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	performSIGN_EXTEND_INREGCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	simplifyOp_VL (SDNode *N)

static SDValue	combineOp_VLToVWOp_VL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
	Combine a binary or FMA operation to its equivalent VW or VW_W form.

static SDValue	combineVWADDSUBWSelect (SDNode *N, SelectionDAG &DAG)

static SDValue	performVWADDSUBW_VLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	tryMemPairCombine (SelectionDAG &DAG, LSBaseSDNode LSNode1, LSBaseSDNode LSNode2, SDValue BasePtr, uint64_t Imm)

static SDValue	performMemPairCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)

static SDValue	performFP_TO_INTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	performFP_TO_INT_SATCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	performBITREVERSECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performVP_REVERSECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performVP_STORECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performVP_TRUNCATECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static unsigned	negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc)

static SDValue	combineVFMADD_VLWithVFNEG_VL (SDNode *N, SelectionDAG &DAG)

static SDValue	performVFMADD_VLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static SDValue	performSRACombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	tryDemorganOfBooleanCondition (SDValue Cond, SelectionDAG &DAG)

static bool	combine_CC (SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	tryFoldSelectIntoOp (SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)

static SDValue	foldSelectOfCTTZOrCTLZ (SDNode *N, SelectionDAG &DAG)

static SDValue	useInversedSetcc (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static bool	matchSelectAddSub (SDValue TrueVal, SDValue FalseVal, bool &SwapCC)

static SDValue	performVSELECTCombine (SDNode *N, SelectionDAG &DAG)
	Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).

static SDValue	performSELECTCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performBUILD_VECTORCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
	If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily the same constant on all lanes), form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).

static MVT	getQDOTXResultType (MVT OpVT)

static SDValue	getZeroPaddedAdd (const SDLoc &DL, SDValue A, SDValue B, SelectionDAG &DAG)
	Given fixed length vectors A and B with equal element types, but possibly different number of elements, return A + B where either A or B is zero padded to the larger number of elements.

static SDValue	foldReduceOperandViaVQDOT (SDValue InVec, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)

static SDValue	performVECREDUCECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)

static SDValue	performINSERT_VECTOR_ELTCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)

static SDValue	performCONCAT_VECTORSCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)

static SDValue	performVECTOR_SHUFFLECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)

static SDValue	combineToVWMACC (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineVqdotAccum (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static bool	legalizeScatterGatherIndexType (SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)

static bool	matchIndexAsShuffle (EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
	Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangement if possible.

static bool	matchIndexAsWiderOp (EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
	Match the index of a gather or scatter operation as an operation with twice the element width and half the number of elements.

static SDValue	combineTruncOfSraSext (SDNode *N, SelectionDAG &DAG)

static SDValue	combineTruncToVnclip (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	combineToVCPOP (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)

static SDValue	performSHLCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)

static uint64_t	computeGREVOrGORC (uint64_t x, unsigned ShAmt, bool IsGORC)

static MachineBasicBlock *	emitReadCounterWidePseudo (MachineInstr &MI, MachineBasicBlock *BB)

static MachineBasicBlock *	emitSplitF64Pseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)

static MachineBasicBlock *	emitBuildPairF64Pseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)

static MachineBasicBlock *	emitQuietFCMP (MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)

static MachineBasicBlock *	EmitLoweredCascadedSelect (MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)

static MachineBasicBlock *	emitSelectPseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)

static const RISCV::RISCVMaskedPseudoInfo *	lookupMaskedIntrinsic (uint16_t MCOpcode, RISCVVType::VLMUL LMul, unsigned SEW)

static MachineBasicBlock *	emitVFROUND_NOEXCEPT_MASK (MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)

static MachineBasicBlock *	emitFROUND (MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)

static SDValue	convertLocVTToValVT (SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL, const RISCVSubtarget &Subtarget)

static SDValue	unpackFromRegLoc (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL, const ISD::InputArg &In, const RISCVTargetLowering &TLI)

static SDValue	convertValVTToLocVT (SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL, const RISCVSubtarget &Subtarget)

static SDValue	unpackFromMemLoc (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)

static SDValue	unpackF64OnRV32DSoftABI (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)

static Align	getPrefTypeAlign (EVT VT, SelectionDAG &DAG)

static Intrinsic::ID	getIntrinsicForMaskedAtomicRMWBinOp (unsigned XLen, AtomicRMWInst::BinOp BinOp)

static Value *	useTpOffset (IRBuilderBase &IRB, unsigned Offset)

Variables
static cl::opt< unsigned >	ExtensionMaxWebSize (DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))

static cl::opt< bool >	AllowSplatInVW_W (DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))

static cl::opt< unsigned >	NumRepeatedDivisors (DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))

static cl::opt< int >	FPImmCost (DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))

static cl::opt< bool >	ReassocShlAddiAdd ("reassoc-shl-addi-add", cl::Hidden, cl::desc("Swap add and addi in cases where the add may " "be combined with a shift"), cl::init(true))

const uint64_t	ModeMask64 = ~RISCVExceptFlags::ALL

const uint32_t	ModeMask32 = ~RISCVExceptFlags::ALL

Macro Definition Documentation

◆ CC_VLS_CASE

#define CC_VLS_CASE ( ABI_VLEN ) case CallingConv::RISCV_VLSCall_##ABI_VLEN:

◆ DEBUG_TYPE

#define DEBUG_TYPE "riscv-lower"

Definition at line 55 of file RISCVISelLowering.cpp.

◆ GET_REGISTER_MATCHER

#define GET_REGISTER_MATCHER

Definition at line 24571 of file RISCVISelLowering.cpp.

◆ GET_RISCVVIntrinsicsTable_IMPL

#define GET_RISCVVIntrinsicsTable_IMPL

Definition at line 24790 of file RISCVISelLowering.cpp.

◆ OP_CASE

#define OP_CASE ( NODE )

Value:

case ISD::NODE: \

return RISCVISD::NODE##_VL;

◆ VP_CASE

#define VP_CASE ( NODE )

Value:

case ISD::VP_##NODE: \

return RISCVISD::NODE##_VL;

NODE

#define NODE(NodeKind)

Definition: ItaniumDemangle.h:2649

Function Documentation

◆ combine_CC()

static bool combine_CC	(	SDValue &	LHS,
		SDValue &	RHS,
		SDValue &	CC,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18536 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::Constant, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueSizeInBits(), llvm::RISCVSubtarget::getXLenVT(), llvm::Value::hasOneUse(), llvm::ISD::isIntEqualitySetCC(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::isPowerOf2_64(), LHS, llvm::Log2_64(), llvm::SelectionDAG::MaskedValueIsZero(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETLT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, translateSetCCForBranch(), tryDemorganOfBooleanCondition(), and llvm::ISD::XOR.

◆ combineAddOfBooleanXor()

static SDValue combineAddOfBooleanXor	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 15597 of file RISCVISelLowering.cpp.

References DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by performADDCombine().

◆ combineBinOpOfExtractToReduceTree()

static SDValue combineBinOpOfExtractToReduceTree	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed by scalar reduction as a vector reduction node.

This exists to recover from a deficiency in SLP which can't handle forests with multiple roots sharing common nodes. In some cases, one of the trees will be vectorized, and the other will remain (unprofitably) scalarized.

Definition at line 15073 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SelectionDAG::getContext(), llvm::RISCVSubtarget::getELen(), llvm::SelectionDAG::getExtractSubvector(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::getVecReduceBaseOpcode(), getVecReduceOpcode(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Value::hasOneUse(), llvm::RISCVSubtarget::hasVInstructions(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::EVT::isScalableVector(), LHS, N, llvm::SelectionDAG::NewNodesMustHaveLegalTypes, Opc, RHS, and std::swap().

Referenced by performADDCombine(), performANDCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), performORCombine(), and performXORCombine().

◆ combineBinOpOfZExt()

static SDValue combineBinOpOfZExt	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

◆ combineBinOpToReduce()

static SDValue combineBinOpToReduce	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15166 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FMAXNUM, llvm::ISD::FMINNUM, llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNeutralConstant(), isNonZeroAVL(), llvm::isNullConstant(), llvm::SDNode::isUndef(), llvm_unreachable, lowerScalarInsert(), N, Opc, llvm::ISD::OR, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::ISD::XOR.

Referenced by performADDCombine(), performANDCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), performORCombine(), and performXORCombine().

◆ combineDeMorganOfBoolean()

static SDValue combineDeMorganOfBoolean	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 15778 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, Opc, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by performANDCombine(), and performORCombine().

◆ combineOp_VLToVWOp_VL()

static SDValue combineOp_VLToVWOp_VL	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Combine a binary or FMA operation to its equivalent VW or VW_W form.

Definition at line 17572 of file RISCVISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorBase< Size_T >::empty(), ExtensionMaxWebSize, llvm::Use::getOperandNo(), llvm::Use::getUser(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), LHS, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::reserve(), RHS, llvm::SmallVectorBase< Size_T >::size(), and std::swap().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine(), performSHLCombine(), performVFMADD_VLCombine(), and performVWADDSUBW_VLCombine().

◆ combineOrOfCZERO()

static SDValue combineOrOfCZERO	(	SDNode *	N,
		SDValue	N0,
		SDValue	N1,
		SelectionDAG &	DAG
	)

static

Definition at line 16061 of file RISCVISelLowering.cpp.

References assert(), Cond, llvm::SDNodeFlags::Disjoint, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), N, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by performORCombine().

◆ combineSelectAndUse()

static SDValue combineSelectAndUse	(	SDNode *	N,
		SDValue	Slct,
		SDValue	OtherOp,
		SelectionDAG &	DAG,
		bool	AllOnes,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15400 of file RISCVISelLowering.cpp.

References llvm::AllOnes, llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::RISCVSubtarget::hasConditionalMoveFusion(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::EVT::isVector(), isZeroOrAllOnes(), N, llvm::ISD::SELECT, and std::swap().

◆ combineSelectAndUseCommutative()

static SDValue combineSelectAndUseCommutative	(	SDNode *	N,
		SelectionDAG &	DAG,
		bool	AllOnes,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15465 of file RISCVISelLowering.cpp.

References llvm::AllOnes, combineSelectAndUse(), and N.

◆ combineSelectToBinOp()

static SDValue combineSelectToBinOp	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ combineShlAddIAdd()

static SDValue combineShlAddIAdd	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15368 of file RISCVISelLowering.cpp.

References combineShlAddIAddImpl(), llvm::RISCVSubtarget::getXLenVT(), N, llvm::Other, and ReassocShlAddiAdd.

Referenced by performADDCombine().

◆ combineShlAddIAddImpl()

static SDValue combineShlAddIAddImpl	(	SDNode *	N,
		SDValue	AddI,
		SDValue	Other,
		SelectionDAG &	DAG
	)

static

◆ combineSubOfBoolean()

static SDValue combineSubOfBoolean	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 15648 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::APInt::isSignedIntN(), N, llvm::ISD::SETCC, and llvm::ISD::XOR.

Referenced by performSUBCombine().

◆ combineSubShiftToOrcB()

static SDValue combineSubShiftToOrcB	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15692 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::RISCVSubtarget::getXLenVT(), llvm::SDValue::hasOneUse(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by performSUBCombine().

◆ combineToVCPOP()

static SDValue combineToVCPOP	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 19771 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::RISCVTargetLowering::computeVLMAX(), convertToScalableVector(), llvm::ISD::CTPOP, DL, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::EVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::RISCVSubtarget::getRealMaxVLen(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::RISCVSubtarget::getXLenVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::isFixedLengthVector(), llvm::EVT::isScalarInteger(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::Log2_32(), N, Opc, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ combineToVWMACC()

static SDValue combineToVWMACC	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 19337 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, getDefaultScalableVLOps(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SDValue::isUndef(), N, Opc, and std::swap().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ combineTruncOfSraSext()

static SDValue combineTruncOfSraSext	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 19581 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::EVT::isVector(), N, llvm::ISD::SIGN_EXTEND, llvm::SMin, llvm::ISD::SMIN, llvm::ISD::SRA, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ combineTruncSelectToSMaxUSat()

static SDValue combineTruncSelectToSMaxUSat	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

◆ combineTruncToVnclip()

static SDValue combineTruncToVnclip	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ combineVectorMulToSraBitcast()

static SDValue combineVectorMulToSraBitcast	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

◆ combineVectorSizedSetCCEquality()

static SDValue combineVectorSizedSetCCEquality	(	EVT	VT,
		SDValue	X,
		SDValue	Y,
		ISD::CondCode	CC,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Try to map an integer comparison with size > XLEN to vector instructions before type legalization splits it up into chunks.

Definition at line 16564 of file RISCVISelLowering.cpp.

Referenced by combineSetCC(), and performSETCCCombine().

◆ combineVFMADD_VLWithVFNEG_VL()

static SDValue combineVFMADD_VLWithVFNEG_VL	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 18306 of file RISCVISelLowering.cpp.

References A, B, llvm::CallingConv::C, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), N, negateFMAOpcode(), and llvm::Offset.

Referenced by performVFMADD_VLCombine().

◆ combineVqdotAccum()

static SDValue combineVqdotAccum	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 19402 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, getDefaultScalableVLOps(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::isUndef(), N, Opc, and std::swap().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ combineVWADDSUBWSelect()

static SDValue combineVWADDSUBWSelect	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 17677 of file RISCVISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullOrNullSplat(), llvm::SDValue::isUndef(), N, Opc, llvm::ISD::VSELECT, X, and Y.

Referenced by performVWADDSUBW_VLCombine().

◆ computeGREVOrGORC()

static uint64_t computeGREVOrGORC	(	uint64_t	x,
		unsigned	ShAmt,
		bool	IsGORC
	)

static

Definition at line 21294 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::computeKnownBitsForTargetNode(), and llvm::RISCVTargetLowering::SimplifyDemandedBitsForTargetNode().

◆ convertFromScalableVector()

static SDValue convertFromScalableVector	(	EVT	VT,
		SDValue	V,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2921 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getExtractSubvector(), and llvm::EVT::isFixedLengthVector().

◆ convertLocVTToValVT()

static SDValue convertLocVTToValVT	(	SelectionDAG &	DAG,
		SDValue	Val,
		const CCValAssign &	VA,
		const SDLoc &	DL,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 22453 of file RISCVISelLowering.cpp.

References llvm::CCValAssign::BCvt, llvm::ISD::BITCAST, convertFromScalableVector(), DL, llvm::CCValAssign::Full, llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getNode(), llvm::CCValAssign::getValVT(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isInteger(), llvm::MVT::isScalableVector(), llvm_unreachable, and llvm::CCValAssign::needsCustom().

◆ convertToScalableVector()

static SDValue convertToScalableVector	(	EVT	VT,
		SDValue	V,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2910 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getUNDEF(), and llvm::EVT::isScalableVector().

◆ convertValVTToLocVT()

static SDValue convertValVTToLocVT	(	SelectionDAG &	DAG,
		SDValue	Val,
		const CCValAssign &	VA,
		const SDLoc &	DL,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 22514 of file RISCVISelLowering.cpp.

References llvm::CCValAssign::BCvt, llvm::ISD::BITCAST, convertToScalableVector(), DL, llvm::CCValAssign::Full, llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getNode(), llvm::CCValAssign::getValVT(), llvm::MVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm_unreachable, and llvm::CCValAssign::needsCustom().

◆ customLegalizeToWOp()

static SDValue customLegalizeToWOp	(	SDNode *	N,
		SelectionDAG &	DAG,
		unsigned	ExtOpc = `ISD::ANY_EXTEND`
	)

static

Definition at line 14278 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), getRISCVWOpcode(), N, and llvm::ISD::TRUNCATE.

◆ customLegalizeToWOpWithSExt()

static SDValue customLegalizeToWOpWithSExt	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Definition at line 14291 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getValueType(), N, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::TRUNCATE.

◆ emitBuildPairF64Pseudo()

static MachineBasicBlock * emitBuildPairF64Pseudo	(	MachineInstr &	MI,
		MachineBasicBlock *	BB,
		const RISCVSubtarget &	Subtarget
	)

static

◆ emitFROUND()

static MachineBasicBlock * emitFROUND	(	MachineInstr &	MI,
		MachineBasicBlock *	MBB,
		const RISCVSubtarget &	Subtarget
	)

static

◆ EmitLoweredCascadedSelect()

static MachineBasicBlock * EmitLoweredCascadedSelect	(	MachineInstr &	First,
		MachineInstr &	Second,
		MachineBasicBlock *	ThisMBB,
		const RISCVSubtarget &	Subtarget
	)

static

◆ emitQuietFCMP()

static MachineBasicBlock * emitQuietFCMP	(	MachineInstr &	MI,
		MachineBasicBlock *	BB,
		unsigned	RelOpcode,
		unsigned	EqOpcode,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 21790 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::TargetSubtargetInfo::getInstrInfo(), llvm::getKillRegState(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::RegState::Kill, MI, MRI, llvm::MachineInstr::NoFPExcept, llvm::MachineInstr::setFlag(), and TII.

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

◆ emitReadCounterWidePseudo()

static MachineBasicBlock * emitReadCounterWidePseudo	(	MachineInstr &	MI,
		MachineBasicBlock *	BB
	)

static

◆ emitSelectPseudo()

static MachineBasicBlock * emitSelectPseudo	(	MachineInstr &	MI,
		MachineBasicBlock *	BB,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 21930 of file RISCVISelLowering.cpp.

◆ emitSplitF64Pseudo()

static MachineBasicBlock * emitSplitF64Pseudo	(	MachineInstr &	MI,
		MachineBasicBlock *	BB,
		const RISCVSubtarget &	Subtarget
	)

static

◆ emitVFROUND_NOEXCEPT_MASK()

static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK	(	MachineInstr &	MI,
		MachineBasicBlock *	BB,
		unsigned	CVTXOpc
	)

static

◆ expandMul()

static SDValue expandMul	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

◆ expandMulToAddOrSubOfShl()

static SDValue expandMulToAddOrSubOfShl	(	SDNode *	N,
		SelectionDAG &	DAG,
		uint64_t	MulAmt
	)

static

Definition at line 16205 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isPowerOf2_64(), llvm::Log2_64(), N, llvm::ISD::SHL, and llvm::ISD::SUB.

Referenced by expandMul().

◆ expandMulToNAFSequence()

static SDValue expandMulToNAFSequence	(	SDNode *	N,
		SelectionDAG &	DAG,
		uint64_t	MulAmt
	)

static

Definition at line 16180 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getShiftAmountConstant(), I, N, llvm::ISD::SHL, and llvm::ISD::SUB.

Referenced by expandMul().

◆ foldBinOpIntoSelectIfProfitable()

static SDValue foldBinOpIntoSelectIfProfitable	(	SDNode *	BO,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 9122 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SDNode::getAsAPIntVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::APInt::isAllOnes(), llvm::ConstantSDNode::isOpaque(), llvm::APInt::isZero(), llvm::ISD::SELECT, and std::swap().

◆ foldConcatVector()

static SDValue foldConcatVector	(	SDValue	V1,
		SDValue	V2
	)

static

If concat_vector(V1,V2) could be folded away to some existing vector source, return it.

Note that the source may be larger than the requested concat_vector (i.e. a extract_subvector might be required.)

Definition at line 4728 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), and llvm::EVT::isScalableVector().

Referenced by getSingleShuffleSrc(), and lowerVECTOR_SHUFFLE().

◆ foldReduceOperandViaVQDOT()

static SDValue foldReduceOperandViaVQDOT	(	SDValue	InVec,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

Definition at line 18980 of file RISCVISelLowering.cpp.

References A, llvm::ISD::ADD, B, DL, foldReduceOperandViaVQDOT(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), getQDOTXResultType(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), getZeroPaddedAdd(), llvm::ISD::isExtOpcode(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isKnownMultipleOf(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MUL, Opc, llvm::ISD::PARTIAL_REDUCE_SMLA, llvm::ISD::PARTIAL_REDUCE_SUMLA, llvm::ISD::PARTIAL_REDUCE_UMLA, llvm::ISD::SIGN_EXTEND, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by foldReduceOperandViaVQDOT(), and performVECREDUCECombine().

◆ foldSelectOfCTTZOrCTLZ()

static SDValue foldSelectOfCTTZOrCTLZ	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

◆ getAllOnesMask()

static SDValue getAllOnesMask	(	MVT	VecVT,
		SDValue	VL,
		const SDLoc &	DL,
		SelectionDAG &	DAG
	)

static

Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.

Definition at line 2942 of file RISCVISelLowering.cpp.

References DL, getMaskTypeFor(), and llvm::SelectionDAG::getNode().

Referenced by getDefaultScalableVLOps(), getDefaultVLOps(), lowerVectorIntrinsicScalars(), and lowerVZIP().

◆ getContainerForFixedLengthVector() [1/2]

static MVT getContainerForFixedLengthVector	(	const TargetLowering &	TLI,
		MVT	VT,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2864 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVSubtarget::getELen(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalableVectorVT(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::isFixedLengthVector(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::RISCV::RVVBitsPerBlock, llvm::MVT::SimpleTy, and useRVVForFixedLengthVectorVT().

◆ getContainerForFixedLengthVector() [2/2]

static MVT getContainerForFixedLengthVector	(	SelectionDAG &	DAG,
		MVT	VT,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2899 of file RISCVISelLowering.cpp.

References getContainerForFixedLengthVector(), and llvm::SelectionDAG::getTargetLoweringInfo().

◆ getDefaultScalableVLOps()

static std::pair< SDValue, SDValue > getDefaultScalableVLOps	(	MVT	VecVT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2949 of file RISCVISelLowering.cpp.

References assert(), DL, getAllOnesMask(), llvm::SelectionDAG::getRegister(), llvm::RISCVSubtarget::getXLenVT(), and llvm::MVT::isScalableVector().

Referenced by combineToVWMACC(), combineVqdotAccum(), getDefaultVLOps(), lowerVECTOR_SHUFFLE(), and performSHLCombine().

◆ getDefaultVLOps() [1/2]

static std::pair< SDValue, SDValue > getDefaultVLOps	(	MVT	VecVT,
		MVT	ContainerVT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2971 of file RISCVISelLowering.cpp.

References assert(), DL, getDefaultScalableVLOps(), getDefaultVLOps(), llvm::MVT::getVectorNumElements(), llvm::MVT::isFixedLengthVector(), and llvm::MVT::isScalableVector().

◆ getDefaultVLOps() [2/2]

static std::pair< SDValue, SDValue > getDefaultVLOps	(	uint64_t	NumElts,
		MVT	ContainerVT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 2958 of file RISCVISelLowering.cpp.

References assert(), DL, getAllOnesMask(), llvm::SelectionDAG::getConstant(), llvm::RISCVSubtarget::getXLenVT(), and llvm::MVT::isScalableVector().

Referenced by combineToVCPOP(), getDefaultVLOps(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCttzElts(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerVZIP(), matchSplatAsGather(), performFP_TO_INTCombine(), and llvm::RISCVTargetLowering::ReplaceNodeResults().

◆ getDeinterleaveShiftAndTrunc()

static SDValue getDeinterleaveShiftAndTrunc	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	Src,
		unsigned	Factor,
		unsigned	Index,
		SelectionDAG &	DAG
	)

static

Definition at line 4885 of file RISCVISelLowering.cpp.

References llvm::MVT::changeVectorElementType(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::divideCoefficientBy(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getInsertSubvector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::ISD::SRL, and llvm::ISD::TRUNCATE.

Referenced by lowerVECTOR_SHUFFLE().

◆ getExactInteger()

static std::optional< APInt > getExactInteger	(	const APFloat &	APF,
		uint32_t	BitWidth
	)

static

Definition at line 3598 of file RISCVISelLowering.cpp.

References llvm::BitWidth, llvm::APFloat::convertToInteger(), llvm::APInt::extractBits(), llvm::APFloatBase::opInvalidOp, and llvm::TowardZero.

Referenced by isSimpleVIDSequence().

◆ getIntrinsicForMaskedAtomicRMWBinOp()

static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp	(	unsigned	XLen,
		AtomicRMWInst::BinOp	BinOp
	)

static

Definition at line 23841 of file RISCVISelLowering.cpp.

References llvm::AtomicRMWInst::Add, llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, and llvm::AtomicRMWInst::Xchg.

◆ getLargeExternalSymbol()

static SDValue getLargeExternalSymbol	(	ExternalSymbolSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG
	)

static

Definition at line 8776 of file RISCVISelLowering.cpp.

References llvm::RISCVConstantPoolValue::Create(), DL, llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstantPool(), and N.

Referenced by llvm::RISCVTargetLowering::LowerCall().

◆ getLargeGlobalAddress()

static SDValue getLargeGlobalAddress	(	GlobalAddressSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG
	)

static

Definition at line 8766 of file RISCVISelLowering.cpp.

References llvm::RISCVConstantPoolValue::Create(), DL, llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstantPool(), and N.

Referenced by llvm::RISCVTargetLowering::LowerCall().

◆ getMaskTypeFor()

static MVT getMaskTypeFor ( MVT VecVT )

static

Return the type of the mask type suitable for masking the provided vector type.

This is simply an i1 element type vector of the same (possibly scalable) length.

Definition at line 2934 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorVT(), and llvm::MVT::isVector().

◆ getPACKOpcode()

static unsigned getPACKOpcode	(	unsigned	DestBW,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 4215 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVSubtarget::is64Bit(), and llvm_unreachable.

Referenced by lowerBuildVectorViaPacking().

◆ getPrefTypeAlign()

static Align getPrefTypeAlign	(	EVT	VT,
		SelectionDAG &	DAG
	)

static

Definition at line 22876 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::DataLayout::getPrefTypeAlign(), and llvm::EVT::getTypeForEVT().

◆ getQDOTXResultType()

static MVT getQDOTXResultType ( MVT OpVT )

static

Definition at line 18953 of file RISCVISelLowering.cpp.

References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::divideCoefficientBy(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isKnownMultipleOf().

Referenced by foldReduceOperandViaVQDOT().

◆ getRISCVVLOp()

static unsigned getRISCVVLOp ( SDValue Op )

static

Get a RISC-V target specified VL op for a given SDNode.

Definition at line 6992 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::ISD::CTTZ_ZERO_UNDEF, llvm::ISD::FMA, llvm::ISD::FMAXIMUMNUM, llvm::ISD::FMAXNUM, llvm::ISD::FMINIMUMNUM, llvm::ISD::FMINNUM, llvm::ISD::LLRINT, llvm_unreachable, llvm::ISD::LRINT, OP_CASE, llvm::ISD::OR, ROTR, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::STRICT_FMA, VP_CASE, llvm::ISD::VSELECT, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

◆ getRISCVWOpcode()

static unsigned getRISCVWOpcode ( unsigned Opcode )

static

Definition at line 14250 of file RISCVISelLowering.cpp.

References llvm_unreachable, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SDIV, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::UDIV, and llvm::ISD::UREM.

◆ getRVVFPReductionOpAndOperands()

static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands	(	SDValue	Op,
		SelectionDAG &	DAG,
		EVT	EltVT,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 11481 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getExtractVectorElt(), llvm_unreachable, llvm::ISD::VECREDUCE_FADD, llvm::ISD::VECREDUCE_FMAX, llvm::ISD::VECREDUCE_FMAXIMUM, llvm::ISD::VECREDUCE_FMIN, llvm::ISD::VECREDUCE_FMINIMUM, and llvm::ISD::VECREDUCE_SEQ_FADD.

◆ getRVVReductionOp()

static unsigned getRVVReductionOp ( unsigned ISDOpcode )

static

Definition at line 11269 of file RISCVISelLowering.cpp.

References llvm_unreachable, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, and llvm::ISD::VECREDUCE_XOR.

◆ getSingleShuffleSrc()

static SDValue getSingleShuffleSrc	(	MVT	VT,
		SDValue	V1,
		SDValue	V2
	)

static

Definition at line 4751 of file RISCVISelLowering.cpp.

References foldConcatVector(), llvm::MVT::getVectorNumElements(), and llvm::SDValue::isUndef().

Referenced by lowerVECTOR_SHUFFLE().

◆ getSmallestVTForIndex()

static std::optional< MVT > getSmallestVTForIndex	(	MVT	VecVT,
		unsigned	MaxIdx,
		SDLoc	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 10012 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::bitsGT(), llvm::MVT::getDoubleNumVectorElementsVT(), llvm::RISCVTargetLowering::getM1VT(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::isScalableVector(), and llvm::MVT::isValid().

◆ getTargetNode() [1/4]

static SDValue getTargetNode	(	BlockAddressSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG,
		unsigned	Flags
	)

static

Definition at line 8749 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetBlockAddress(), and N.

◆ getTargetNode() [2/4]

static SDValue getTargetNode	(	ConstantPoolSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG,
		unsigned	Flags
	)

static

Definition at line 8755 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetConstantPool(), and N.

◆ getTargetNode() [3/4]

static SDValue getTargetNode	(	GlobalAddressSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG,
		unsigned	Flags
	)

static

Definition at line 8744 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getTargetGlobalAddress(), and N.

◆ getTargetNode() [4/4]

static SDValue getTargetNode	(	JumpTableSDNode *	N,
		const SDLoc &	DL,
		EVT	Ty,
		SelectionDAG &	DAG,
		unsigned	Flags
	)

static

Definition at line 8761 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetJumpTable(), and N.

◆ getVCIXISDNodeVOID()

static SDValue getVCIXISDNodeVOID	(	SDValue &	Op,
		SelectionDAG &	DAG,
		unsigned	Type
	)

inlinestatic

Definition at line 10973 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getNode(), Operands, and processVCIXOperands().

◆ getVCIXISDNodeWCHAIN()

static SDValue getVCIXISDNodeWCHAIN	(	SDValue &	Op,
		SelectionDAG &	DAG,
		unsigned	Type
	)

inlinestatic

Definition at line 10936 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), DL, llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::MachineFunction::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), Operands, and processVCIXOperands().

◆ getVecReduceOpcode()

static unsigned getVecReduceOpcode ( unsigned Opc )

static

Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.

Definition at line 15040 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::FADD, llvm_unreachable, Opc, llvm::ISD::OR, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_FADD, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.

Referenced by combineBinOpOfExtractToReduceTree().

◆ getVLOperand()

static SDValue getVLOperand ( SDValue Op )

static

Definition at line 2775 of file RISCVISelLowering.cpp.

References assert(), II, llvm::ISD::INTRINSIC_W_CHAIN, and llvm::ISD::INTRINSIC_WO_CHAIN.

Referenced by lowerVectorIntrinsicScalars().

◆ getVSlidedown()

static SDValue getVSlidedown	(	SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const SDLoc &	DL,
		EVT	VT,
		SDValue	Passthru,
		SDValue	Op,
		SDValue	Offset,
		SDValue	Mask,
		SDValue	VL,
		unsigned	Policy = `RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED`
	)

static

Definition at line 3569 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::SDValue::isUndef(), llvm::RISCVVType::MASK_AGNOSTIC, llvm::Offset, and llvm::RISCVVType::TAIL_AGNOSTIC.

Referenced by lowerBUILD_VECTOR(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlidedown(), and llvm::RISCVTargetLowering::ReplaceNodeResults().

◆ getVSlideup()

static SDValue getVSlideup	(	SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const SDLoc &	DL,
		EVT	VT,
		SDValue	Passthru,
		SDValue	Op,
		SDValue	Offset,
		SDValue	Mask,
		SDValue	VL,
		unsigned	Policy = `RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED`
	)

static

Definition at line 3581 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::SDValue::isUndef(), llvm::RISCVVType::MASK_AGNOSTIC, llvm::Offset, and llvm::RISCVVType::TAIL_AGNOSTIC.

Referenced by lowerVECTOR_SHUFFLE(), and lowerVECTOR_SHUFFLEAsVSlideup().

◆ getWideningInterleave()

static SDValue getWideningInterleave	(	SDValue	EvenV,
		SDValue	OddV,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 5251 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getWideningSpread(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::SDValue::isUndef(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy().

Referenced by lowerVECTOR_SHUFFLE().

◆ getWideningSpread()

static SDValue getWideningSpread	(	SDValue	V,
		unsigned	Factor,
		unsigned	Index,
		const SDLoc &	DL,
		SelectionDAG &	DAG
	)

static

Definition at line 5226 of file RISCVISelLowering.cpp.

References llvm::MVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::ISD::SHL, and llvm::ISD::ZERO_EXTEND.

Referenced by getWideningInterleave(), and lowerVECTOR_SHUFFLE().

◆ getZeroPaddedAdd()

static SDValue getZeroPaddedAdd	(	const SDLoc &	DL,
		SDValue	A,
		SDValue	B,
		SelectionDAG &	DAG
	)

static

Given fixed length vectors A and B with equal element types, but possibly different number of elements, return A + B where either A or B is zero padded to the larger number of elements.

Definition at line 18962 of file RISCVISelLowering.cpp.

References A, llvm::ISD::ADD, assert(), B, DL, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), and std::swap().

Referenced by foldReduceOperandViaVQDOT().

◆ isAlternating()

static bool isAlternating	(	const std::array< std::pair< int, int >, 2 > &	SrcInfo,
		ArrayRef< int >	Mask,
		unsigned	Factor,
		bool	RequiredPolarity
	)

static

Definition at line 4834 of file RISCVISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::enumerate(), and Idx.

Referenced by isZipEven(), and isZipOdd().

◆ isCompressMask()

static bool isCompressMask ( ArrayRef< int > Mask )

static

Definition at line 5569 of file RISCVISelLowering.cpp.

References llvm::enumerate(), Idx, and llvm::Last.

Referenced by lowerVECTOR_SHUFFLE().

◆ isElementRotate()

static bool isElementRotate	(	const std::array< std::pair< int, int >, 2 > &	SrcInfo,
		unsigned	NumElts
	)

static

Definition at line 4826 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::isShuffleMaskLegal(), and lowerVECTOR_SHUFFLE().

◆ isInterleaveShuffle()

static bool isInterleaveShuffle	(	ArrayRef< int >	Mask,
		MVT	VT,
		int &	EvenSrc,
		int &	OddSrc,
		const RISCVSubtarget &	Subtarget
	)

static

Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguous elements from another vector into the odd elements.

EvenSrc will contain the element that should be in the first even element. OddSrc will contain the element that should be in the first odd element. These can be the first element in a source or the element half way through the source.

Definition at line 4772 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::ShuffleVectorInst::isInterleaveMask(), and Size.

Referenced by llvm::RISCVTargetLowering::isShuffleMaskLegal(), and lowerVECTOR_SHUFFLE().

◆ isLegalBitRotate()

static bool isLegalBitRotate	(	ArrayRef< int >	Mask,
		EVT	VT,
		const RISCVSubtarget &	Subtarget,
		MVT &	RotateVT,
		unsigned &	RotateAmt
	)

static

Definition at line 5394 of file RISCVISelLowering.cpp.

References llvm::MVT::getIntegerVT(), llvm::EVT::getScalarSizeInBits(), llvm::RISCVSubtarget::getTargetLowering(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ShuffleVectorInst::isBitRotateMask(), and llvm::TargetLoweringBase::isTypeLegal().

Referenced by lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), and lowerVECTOR_SHUFFLEAsRotate().

◆ isLocalRepeatingShuffle()

static bool isLocalRepeatingShuffle	(	ArrayRef< int >	Mask,
		int	Span
	)

static

Is this mask local (i.e.

elements only move within their local span), and repeating (that is, the same rearrangement is being done within each span)?

Definition at line 5651 of file RISCVISelLowering.cpp.

References llvm::enumerate(), and I.

Referenced by lowerVECTOR_SHUFFLE().

◆ isLowSourceShuffle()

static bool isLowSourceShuffle	(	ArrayRef< int >	Mask,
		int	Span
	)

static

Is this mask only using elements from the first span of the input?

Definition at line 5668 of file RISCVISelLowering.cpp.

References llvm::all_of(), and Idx.

Referenced by lowerVECTOR_SHUFFLE().

◆ isMaskedSlidePair()

static bool isMaskedSlidePair	(	ArrayRef< int >	Mask,
		std::array< std::pair< int, int >, 2 > &	SrcInfo
	)

static

Is this mask representing a masked combination of two slides?

Definition at line 4807 of file RISCVISelLowering.cpp.

References assert(), llvm::isMaskedSlidePair(), and std::swap().

◆ isNonZeroAVL()

static bool isNonZeroAVL ( SDValue AVL )

static

Definition at line 11392 of file RISCVISelLowering.cpp.

Referenced by combineBinOpToReduce(), and lowerReductionSeq().

◆ isPromotedOpNeedingSplit()

static bool isPromotedOpNeedingSplit	(	SDValue	Op,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 7149 of file RISCVISelLowering.cpp.

References llvm::RISCVSubtarget::hasVInstructionsF16(), and llvm::RISCVSubtarget::hasVInstructionsF16Minimal().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ isSimpleVIDSequence()

static std::optional< VIDSequence > isSimpleVIDSequence	(	SDValue	Op,
		unsigned	EltSizeInBits
	)

static

Definition at line 3631 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::enumerate(), getExactInteger(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::APInt::getSExtValue(), Idx, isConstant(), llvm::APInt::sdiv(), and llvm::APInt::srem().

Referenced by lowerBuildVectorViaVID(), and llvm::RISCVTargetLowering::PerformDAGCombine().

◆ isSpanSplatShuffle()

static bool isSpanSplatShuffle	(	ArrayRef< int >	Mask,
		int	Span
	)

static

Return true for a mask which performs an arbitrary shuffle within the first span, and then repeats that same result across all remaining spans.

Note that this doesn't check if all the inputs come from a single span!

Definition at line 5675 of file RISCVISelLowering.cpp.

References llvm::enumerate(), and I.

Referenced by lowerVECTOR_SHUFFLE().

◆ isValidEGW()

static bool isValidEGW	(	int	EGS,
		EVT	VT,
		const RISCVSubtarget &	Subtarget
	)

inlinestatic

Definition at line 10696 of file RISCVISelLowering.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), and llvm::RISCV::RVVBitsPerBlock.

◆ isValidVisniInsertExtractIndex()

static bool isValidVisniInsertExtractIndex ( SDValue Idx )

static

Definition at line 10033 of file RISCVISelLowering.cpp.

References Idx, and llvm::isNullConstant().

◆ isZipEven()

static bool isZipEven	(	const std::array< std::pair< int, int >, 2 > &	SrcInfo,
		ArrayRef< int >	Mask,
		unsigned &	Factor
	)

static

Given a shuffle which can be represented as a pair of two slides, see if it is a zipeven idiom.

Zipeven is: vs2: a0 a1 a2 a3 vs1: b0 b1 b2 b3 vd: a0 b0 a2 b2

Definition at line 4857 of file RISCVISelLowering.cpp.

References isAlternating(), and llvm::isPowerOf2_32().

Referenced by lowerVECTOR_SHUFFLE().

◆ isZipOdd()

static bool isZipOdd	(	const std::array< std::pair< int, int >, 2 > &	SrcInfo,
		ArrayRef< int >	Mask,
		unsigned &	Factor
	)

static

Given a shuffle which can be represented as a pair of two slides, see if it is a zipodd idiom.

Zipodd is: vs2: a0 a1 a2 a3 vs1: b0 b1 b2 b3 vd: a1 b1 a3 b3 Note that the operand order is swapped due to the way we canonicalize the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.

Definition at line 4872 of file RISCVISelLowering.cpp.

References isAlternating(), and llvm::isPowerOf2_32().

Referenced by lowerVECTOR_SHUFFLE().

◆ legalizeScatterGatherIndexType()

static bool legalizeScatterGatherIndexType	(	SDLoc	DL,
		SDValue &	Index,
		ISD::MemIndexType &	IndexType,
		RISCVTargetLowering::DAGCombinerInfo &	DCI
	)

static

Definition at line 19466 of file RISCVISelLowering.cpp.

References llvm::EVT::bitsLT(), llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MachineFunction::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::UNSIGNED_SCALED.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ lookupMaskedIntrinsic()

static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic	(	uint16_t	MCOpcode,
		RISCVVType::VLMUL	LMul,
		unsigned	SEW
	)

static

Definition at line 22082 of file RISCVISelLowering.cpp.

References assert(), and llvm::Masked.

Referenced by emitVFROUND_NOEXCEPT_MASK().

◆ LowerATOMIC_FENCE()

static SDValue LowerATOMIC_FENCE	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 6693 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::ISD::MEMBARRIER, llvm::SequentiallyConsistent, llvm::SyncScope::SingleThread, and llvm::SyncScope::System.

◆ lowerBitreverseShuffle()

static SDValue lowerBitreverseShuffle	(	ShuffleVectorSDNode *	SVN,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 5344 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BITREVERSE, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertSubvector(), llvm::EVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getSimpleValueType(), llvm::RISCVSubtarget::getTargetLowering(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::ShuffleVectorInst::isReverseMask(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::PowerOf2Ceil(), llvm::ArrayRef< T >::size(), and llvm::ISD::SRL.

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerBUILD_VECTOR()

static SDValue lowerBUILD_VECTOR	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 4292 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerBuildVectorOfConstants()

static SDValue lowerBuildVectorOfConstants	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3975 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsLE(), llvm::MVT::changeVectorElementType(), llvm::SelectionDAG::ComputeMaxSignificantBits(), convertFromScalableVector(), convertToScalableVector(), llvm::divideCeil(), DL, llvm::enumerate(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::RISCVSubtarget::getELen(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertVectorElt(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSignedConstant(), llvm::MVT::getSizeInBits(), llvm::getSplatValue(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLen(), llvm::RISCVSubtarget::getXLenVT(), I, llvm::RISCVSubtarget::is64Bit(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), Opc, OpIdx, llvm::SelectionDAG::shouldOptForSize(), and llvm::Splat.

Referenced by lowerBUILD_VECTOR().

◆ lowerBuildVectorViaDominantValues()

static SDValue lowerBuildVectorViaDominantValues	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large proportion of the elements.

In such cases we can splat a vector with the dominant element and make up the shortfall with INSERT_VECTOR_ELTs. Returns SDValue if not profitable. Note that this includes vectors of 2 elements by association. The upper-most element is the "dominant" one, allowing us to use a splat to "insert" the upper element, and an insert of the lower element at position 0, which improves codegen.

Definition at line 3870 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), llvm::count_if(), DL, llvm::enumerate(), llvm::SelectionDAG::getBuildVector(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::Log2_32(), OpIdx, llvm::SelectionDAG::shouldOptForSize(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::transform(), and llvm::ISD::VSELECT.

Referenced by lowerBUILD_VECTOR(), and lowerBuildVectorOfConstants().

◆ lowerBuildVectorViaPacking()

static SDValue lowerBuildVectorViaPacking	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Double the element size of the build vector to reduce the number of vslide1down in the build vector chain.

In the worst case, this trades three scalar operations for 1 vector operation. Scalar operations are generally lower latency, and for out-of-order cores we also benefit from additional parallelism.

Definition at line 4235 of file RISCVISelLowering.cpp.

References A, llvm::ISD::AND, assert(), B, llvm::ISD::BITCAST, llvm::SDNodeFlags::Disjoint, DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), getPACKOpcode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLen(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::ISD::SHL, and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerBUILD_VECTOR().

◆ lowerBuildVectorViaVID()

static SDValue lowerBuildVectorViaVID	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerConstant()

static SDValue lowerConstant	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 6612 of file RISCVISelLowering.cpp.

References assert(), llvm::SmallVectorBase< Size_T >::empty(), llvm::RISCVMatInt::generateInstSeq(), llvm::RISCVMatInt::generateTwoRegInstSeq(), llvm::RISCVSubtarget::getMaxBuildIntsCost(), llvm::SelectionDAG::shouldOptForSize(), llvm::SmallVectorBase< Size_T >::size(), and llvm::RISCVSubtarget::useConstantPoolForLargeInts().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerCttzElts()

static SDValue lowerCttzElts	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 10614 of file RISCVISelLowering.cpp.

References convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getElementCount(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementCount(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::isOneConstant(), N, and llvm::ISD::SETLT.

Referenced by llvm::RISCVTargetLowering::ReplaceNodeResults().

◆ lowerDisjointIndicesShuffle()

static SDValue lowerDisjointIndicesShuffle	(	ShuffleVectorSDNode *	SVN,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Given a shuffle where the indices are disjoint between the two sources, e.g.:

t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>

Merge the two sources into one and do a single source shuffle:

t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1> t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>

A vselect will either be merged into a masked instruction or be lowered as a vmerge.vvm, which is cheaper than a vrgather.vv.

Definition at line 5600 of file RISCVISelLowering.cpp.

References llvm::MVT::changeVectorElementType(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::RISCVSubtarget::getXLenVT(), I, Idx, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Select, and llvm::ISD::VSELECT.

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerFABSorFNEG()

static SDValue lowerFABSorFNEG	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 6908 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::ISD::FABS, llvm::ISD::FNEG, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::RISCVSubtarget::getXLen(), llvm::RISCVSubtarget::getXLenVT(), and llvm::ISD::XOR.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerFCOPYSIGN()

static SDValue lowerFCOPYSIGN	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerFixedVectorSegLoadIntrinsics()

static SDValue lowerFixedVectorSegLoadIntrinsics	(	unsigned	IntNo,
		SDValue	Op,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

◆ lowerFixedVectorSegStoreIntrinsics()

static SDValue lowerFixedVectorSegStoreIntrinsics	(	unsigned	IntNo,
		SDValue	Op,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

◆ lowerFMAXIMUM_FMINIMUM()

static SDValue lowerFMAXIMUM_FMINIMUM	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerFP_TO_INT()

static SDValue lowerFP_TO_INT	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3221 of file RISCVISelLowering.cpp.

References DL, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::RISCVSubtarget::hasStdExtZfhOrZhinx(), and llvm::ISD::STRICT_FP_EXTEND.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerFP_TO_INT_SAT()

static SDValue lowerFP_TO_INT_SAT	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3108 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_SINT_SAT, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelectCC(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::RISCVSubtarget::getXLenVT(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::RISCVSubtarget::hasStdExtZfhOrZhinx(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), Opc, llvm::RISCVFPRndMode::RTZ, and llvm::ISD::SETUO.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3500 of file RISCVISelLowering.cpp.

References llvm::APFloat::convertFromAPInt(), DL, llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isVector(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), and llvm::SelectionDAG::shouldOptForSize().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerGetVectorLength()

static SDValue lowerGetVectorLength	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 10579 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::RISCVVType::encodeLMUL(), llvm::RISCVVType::encodeSEW(), llvm::RISCVSubtarget::getELen(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isPowerOf2_32(), N, llvm::RISCV::RVVBitsPerBlock, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::ReplaceNodeResults().

◆ lowerINT_TO_FP()

static SDValue lowerINT_TO_FP	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3081 of file RISCVISelLowering.cpp.

References DL, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::RISCVSubtarget::hasStdExtZfhOrZhinx(), and llvm::ISD::STRICT_FP_ROUND.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ LowerPREFETCH()

static SDValue LowerPREFETCH	(	SDValue	Op,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 6682 of file RISCVISelLowering.cpp.

◆ lowerReductionSeq()

static SDValue lowerReductionSeq	(	unsigned	RVVOpcode,
		MVT	ResVT,
		SDValue	StartValue,
		SDValue	Vec,
		SDValue	Mask,
		SDValue	VL,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.

Definition at line 11401 of file RISCVISelLowering.cpp.

References llvm::MVT::bitsLE(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertSubvector(), llvm::RISCVTargetLowering::getM1VT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), isNonZeroAVL(), lowerScalarInsert(), Reduction, and llvm::RISCVVType::TAIL_AGNOSTIC.

◆ lowerScalarInsert()

static SDValue lowerScalarInsert	(	SDValue	Scalar,
		SDValue	VL,
		MVT	VT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerScalarSplat()

static SDValue lowerScalarSplat	(	SDValue	Passthru,
		SDValue	Scalar,
		SDValue	VL,
		MVT	VT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerShuffleViaVRegSplitting()

static SDValue lowerShuffleViaVRegSplitting	(	ShuffleVectorSDNode *	SVN,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ lowerVECTOR_SHUFFLE()

static SDValue lowerVECTOR_SHUFFLE	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 5721 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

◆ lowerVECTOR_SHUFFLEAsRotate()

static SDValue lowerVECTOR_SHUFFLEAsRotate	(	ShuffleVectorSDNode *	SVN,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 5413 of file RISCVISelLowering.cpp.

References llvm::ISD::BSWAP, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDNode::getValueType(), isLegalBitRotate(), and llvm::ISD::ROTL.

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerVECTOR_SHUFFLEAsVRGatherVX()

static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX	(	ShuffleVectorSDNode *	SVN,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

Match a single source shuffle which is an identity except that some particular element is repeated.

This can be lowered as a masked vrgather.vi/vx. Note that the two source form of this is handled by the recursive splitting logic and doesn't need special handling.

Definition at line 4909 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::enumerate(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLenVT(), I, llvm::SDValue::isUndef(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), llvm::Splat, and llvm::ISD::VSELECT.

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerVECTOR_SHUFFLEAsVSlide1()

static SDValue lowerVECTOR_SHUFFLEAsVSlide1	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	V1,
		SDValue	V2,
		ArrayRef< int >	Mask,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

Match v(f)slide1up/down idioms.

These operations involve sliding N-1 elements to make room for an inserted scalar at one end.

Definition at line 5078 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::sampleprof::Base, llvm::MVT::changeVectorElementTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVSubtarget::hasVInstructionsF16(), llvm::MVT::isFloatingPoint(), llvm::Offset, llvm::Splat, and std::swap().

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerVECTOR_SHUFFLEAsVSlidedown()

static SDValue lowerVECTOR_SHUFFLEAsVSlidedown	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	V1,
		SDValue	V2,
		ArrayRef< int >	Mask,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 4958 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getUNDEF(), getVSlidedown(), llvm::RISCVSubtarget::getXLenVT(), llvm::Offset, and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerVECTOR_SHUFFLEAsVSlideup()

static SDValue lowerVECTOR_SHUFFLEAsVSlideup	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	V1,
		SDValue	V2,
		ArrayRef< int >	Mask,
		const RISCVSubtarget &	Subtarget,
		SelectionDAG &	DAG
	)

static

Definition at line 5033 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), getVSlideup(), llvm::RISCVSubtarget::getXLenVT(), llvm::ShuffleVectorInst::isInsertSubvectorMask(), llvm::RISCVVType::MASK_AGNOSTIC, llvm::RISCVVType::TAIL_AGNOSTIC, and llvm::RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED.

Referenced by lowerVECTOR_SHUFFLE().

◆ lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3292 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementTypeToInteger(), llvm::APFloat::convertFromAPInt(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::FCEIL, llvm::ISD::FFLOOR, llvm::ISD::FNEARBYINT, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::ISD::FTRUNC, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstantFP(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getFreeze(), getMaskTypeFor(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVFPRndMode::Invalid, llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), llvm_unreachable, matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), and llvm::ISD::SETOLT.

Referenced by lowerFTRUNC_FCEIL_FFLOOR_FROUND(), and llvm::RISCVTargetLowering::LowerOperation().

◆ lowerVectorIntrinsicScalars()

static SDValue lowerVectorIntrinsicScalars	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 10398 of file RISCVISelLowering.cpp.

◆ lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3399 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementTypeToInteger(), llvm::APFloat::convertFromAPInt(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstantFP(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getFreeze(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVFPRndMode::Invalid, llvm::MVT::isFixedLengthVector(), llvm_unreachable, matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), llvm::ISD::SETOLT, llvm::ISD::STRICT_FCEIL, llvm::ISD::STRICT_FFLOOR, llvm::ISD::STRICT_FNEARBYINT, llvm::ISD::STRICT_FROUND, llvm::ISD::STRICT_FROUNDEVEN, and llvm::ISD::STRICT_FTRUNC.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerVectorXRINT_XROUND()

static SDValue lowerVectorXRINT_XROUND	(	SDValue	Op,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3528 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), DL, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorElementType(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), and matchRoundingOp().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ lowerVZIP()

static SDValue lowerVZIP	(	unsigned	Opc,
		SDValue	Op0,
		SDValue	Op1,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 5178 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::MVT::changeVectorElementTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), DL, getAllOnesMask(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getExtractSubvector(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getInsertSubvector(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::RISCVTargetLowering::getM1VT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorNumElements(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::SDValue::isUndef(), and Opc.

Referenced by lowerVECTOR_SHUFFLE().

◆ matchIndexAsShuffle()

static bool matchIndexAsShuffle	(	EVT	VT,
		SDValue	Index,
		SDValue	Mask,
		SmallVector< int > &	ShuffleMask
	)

static

Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangement if possible.

Will only match if all lanes are touched, and thus replacing the scatter or gather with a unit strided access and shuffle is legal.

Definition at line 19499 of file RISCVISelLowering.cpp.

References llvm::BitVector::all(), assert(), llvm::CallingConv::C, llvm::SmallVectorBase< Size_T >::empty(), llvm::EVT::getScalarStoreSize(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::BitVector::set().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ matchIndexAsWiderOp()

static bool matchIndexAsWiderOp	(	EVT	VT,
		SDValue	Index,
		SDValue	Mask,
		Align	BaseAlign,
		const RISCVSubtarget &	ST
	)

static

Match the index of a gather or scatter operation as an operation with twice the element width and half the number of elements.

This is generally profitable (if legal) because these operations are linear in VL, so even if we cause some extract VTYPE/VL toggles, we still come out ahead.

Definition at line 19534 of file RISCVISelLowering.cpp.

References llvm::CallingConv::C, llvm::EVT::getScalarStoreSize(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVectorAllOnes(), and llvm::Last.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ matchRoundingOp()

static RISCVFPRndMode::RoundingMode matchRoundingOp ( unsigned Opc )

static

Definition at line 3246 of file RISCVISelLowering.cpp.

Referenced by lowerFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), performFP_TO_INT_SATCombine(), and performFP_TO_INTCombine().

◆ matchSelectAddSub()

static bool matchSelectAddSub	(	SDValue	TrueVal,
		SDValue	FalseVal,
		bool &	SwapCC
	)

static

Definition at line 18825 of file RISCVISelLowering.cpp.

References A, llvm::ISD::ADD, B, llvm::ISD::SUB, and std::swap().

Referenced by performVECTOR_SHUFFLECombine(), and performVSELECTCombine().

◆ matchSetCC()

static std::optional< bool > matchSetCC	(	SDValue	LHS,
		SDValue	RHS,
		ISD::CondCode	CC,
		SDValue	Val
	)

static

Definition at line 9023 of file RISCVISelLowering.cpp.

References assert(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCInverse(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), LHS, RHS, and llvm::ISD::SETCC.

◆ matchSplatAsGather()

static SDValue matchSplatAsGather	(	SDValue	SplatVal,
		MVT	VT,
		const SDLoc &	DL,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 3737 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorMinNumElements(), llvm::RISCVSubtarget::getXLenVT(), Idx, llvm::MVT::isFixedLengthVector(), llvm::details::FixedOrScalableQuantity< TypeSize, uint64_t >::isKnownLE(), and llvm::TargetLoweringBase::isTypeLegal().

Referenced by lowerBUILD_VECTOR(), and llvm::RISCVTargetLowering::PerformDAGCombine().

◆ narrowIndex()

static bool narrowIndex	(	SDValue &	N,
		ISD::MemIndexType	IndexType,
		SelectionDAG &	DAG
	)

static

According to the property that indexed load/store instructions zero-extend their indices, try to narrow the type of index operand.

Definition at line 16496 of file RISCVISelLowering.cpp.

References llvm::EVT::bitsLT(), llvm::CallingConv::C, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getRoundIntegerType(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVector(), N, llvm::SDNode::ops(), llvm::PowerOf2Ceil(), llvm::ISD::SHL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ negateFMAOpcode()

static unsigned negateFMAOpcode	(	unsigned	Opcode,
		bool	NegMul,
		bool	NegAcc
	)

static

Definition at line 18268 of file RISCVISelLowering.cpp.

References llvm_unreachable.

Referenced by combineFMA(), combineFMADDSUB(), combineVFMADD_VLWithVFNEG_VL(), and llvm::X86TargetLowering::getNegatedExpression().

◆ performADDCombine()

static SDValue performADDCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15621 of file RISCVISelLowering.cpp.

References combineAddOfBooleanXor(), combineBinOpOfExtractToReduceTree(), combineBinOpOfZExt(), combineBinOpToReduce(), combineSelectAndUseCommutative(), combineShlAddIAdd(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), N, transformAddImmMulImm(), and transformAddShlImm().

◆ performANDCombine()

static SDValue performANDCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performBITREVERSECombine()

static SDValue performBITREVERSECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18040 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BSWAP, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::EVT::isScalarInteger(), and N.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performBUILD_VECTORCombine()

static SDValue performBUILD_VECTORCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily the same constant on all lanes), form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).

We assume that materializing a constant build vector will be no more expensive that performing O(n) binops.

Definition at line 18895 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::TargetLoweringBase::isBinOp(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SelectionDAG::isSafeToSpeculativelyExecute(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performCONCAT_VECTORSCombine()

static SDValue performCONCAT_VECTORSCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

Definition at line 19153 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performFP_TO_INT_SATCombine()

static SDValue performFP_TO_INT_SATCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 17977 of file RISCVISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FP_TO_SINT_SAT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelectCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::RISCVSubtarget::getXLenVT(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::RISCVFPRndMode::Invalid, llvm::TargetLoweringBase::isTypeLegal(), matchRoundingOp(), N, Opc, and llvm::ISD::SETUO.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performFP_TO_INTCombine()

static SDValue performFP_TO_INTCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performINSERT_VECTOR_ELTCombine()

static SDValue performINSERT_VECTOR_ELTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

◆ performMemPairCombine()

static SDValue performMemPairCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

Definition at line 17793 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, getExtensionType(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemoryVT(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MachineFunction::getSubtarget(), llvm::Use::getUser(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isSimple(), llvm::ISD::LOAD, N, Ptr, tryMemPairCombine(), and llvm::SDNode::uses().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performMULCombine()

static SDValue performMULCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 16445 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, combineBinOpOfZExt(), combineVectorMulToSraBitcast(), DL, expandMul(), llvm::SelectionDAG::getNode(), llvm::isOneOrOneSplat(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::SUB, and std::swap().

◆ performORCombine()

static SDValue performORCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 16096 of file RISCVISelLowering.cpp.

References combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineDeMorganOfBoolean(), combineOrOfCZERO(), combineSelectAndUseCommutative(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), and N.

◆ performSELECTCombine()

static SDValue performSELECTCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18872 of file RISCVISelLowering.cpp.

References foldSelectOfCTTZOrCTLZ(), llvm::RISCVSubtarget::hasConditionalMoveFusion(), N, tryFoldSelectIntoOp(), and useInversedSetcc().

◆ performSETCCCombine()

static SDValue performSETCCCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 16625 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, combineVectorSizedSetCCEquality(), Cond, llvm::APInt::countr_zero(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getActiveBits(), llvm::SelectionDAG::getBoolConstant(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSExtValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::APInt::isNegatedPowerOf2(), llvm::isNullConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SETNE, llvm::APInt::sext(), llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRL, and llvm::APInt::trunc().

◆ performSHLCombine()

static SDValue performSHLCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performSIGN_EXTEND_INREGCombine()

static SDValue performSIGN_EXTEND_INREGCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performSRACombine()

static SDValue performSRACombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performSUBCombine()

static SDValue performSUBCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15745 of file RISCVISelLowering.cpp.

References combineBinOpOfZExt(), combineSelectAndUse(), combineSubOfBoolean(), combineSubShiftToOrcB(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, llvm::ISD::SETCC, llvm::ISD::SETLT, and llvm::ISD::SRA.

◆ performTRUNCATECombine()

static SDValue performTRUNCATECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15892 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, combineTruncSelectToSMaxUSat(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), N, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performVECREDUCECombine()

static SDValue performVECREDUCECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

Definition at line 19069 of file RISCVISelLowering.cpp.

References DL, foldReduceOperandViaVQDOT(), llvm::SelectionDAG::getNode(), N, and llvm::ISD::VECREDUCE_ADD.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performVECTOR_SHUFFLECombine()

static SDValue performVECTOR_SHUFFLECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget,
		const RISCVTargetLowering &	TLI
	)

static

Definition at line 19279 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performVFMADD_VLCombine()

static SDValue performVFMADD_VLCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18345 of file RISCVISelLowering.cpp.

References combineOp_VLToVWOp_VL(), combineVFMADD_VLWithVFNEG_VL(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), and N.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performVP_REVERSECombine()

static SDValue performVP_REVERSECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performVP_STORECombine()

static SDValue performVP_STORECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ performVP_TRUNCATECombine()

static SDValue performVP_TRUNCATECombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18185 of file RISCVISelLowering.cpp.

References DL, llvm::drop_end(), llvm::find_if(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), I, llvm::isOneOrOneSplat(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, Operands, llvm::Other, and std::swap().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performVSELECTCombine()

static SDValue performVSELECTCombine	(	SDNode *	N,
		SelectionDAG &	DAG
	)

static

Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).

This allows us match a vadd.vv fed by a masked vrsub, which reduces register pressure over the add followed by masked vsub sequence.

Definition at line 18848 of file RISCVISelLowering.cpp.

References A, llvm::ISD::ADD, B, DL, llvm::SelectionDAG::getLogicalNOT(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDNode::getValueType(), matchSelectAddSub(), N, llvm::Sub, and llvm::ISD::VSELECT.

◆ performVWADDSUBW_VLCombine()

static SDValue performVWADDSUBW_VLCombine	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 17719 of file RISCVISelLowering.cpp.

References assert(), combineOp_VLToVWOp_VL(), combineVWADDSUBWSelect(), llvm::TargetLowering::DAGCombinerInfo::DAG, N, and Opc.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ performXORCombine()

static SDValue performXORCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ processVCIXOperands()

static void processVCIXOperands	(	SDValue &	OrigOp,
		SmallVectorImpl< SDValue > &	Operands,
		SelectionDAG &	DAG
	)

static

Definition at line 10673 of file RISCVISelLowering.cpp.

References convertToScalableVector(), llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getSubtarget(), llvm::MVT::getVectorVT(), Operands, and promoteVCIXScalar().

Referenced by getVCIXISDNodeVOID(), and getVCIXISDNodeWCHAIN().

◆ promoteVCIXScalar()

static void promoteVCIXScalar	(	const SDValue &	Op,
		SmallVectorImpl< SDValue > &	Operands,
		SelectionDAG &	DAG
	)

inlinestatic

Definition at line 10637 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsLT(), DL, llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::MachineFunction::getSubtarget(), llvm::RISCVSubtarget::getXLenVT(), II, llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, llvm::MVT::isScalarInteger(), Operands, and llvm::ISD::SIGN_EXTEND.

Referenced by processVCIXOperands().

◆ reduceANDOfAtomicLoad()

static SDValue reduceANDOfAtomicLoad	(	SDNode *	N,
		TargetLowering::DAGCombinerInfo &	DCI
	)

static

◆ reverseZExtICmpCombine()

static SDValue reverseZExtICmpCombine	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ simplifyOp_VL()

static SDValue simplifyOp_VL ( SDNode * N )

static

Definition at line 17540 of file RISCVISelLowering.cpp.

References A, assert(), B, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::SDValue::isUndef(), and N.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

◆ splatPartsI64WithVL()

static SDValue splatPartsI64WithVL	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	Passthru,
		SDValue	Lo,
		SDValue	Hi,
		SDValue	VL,
		SelectionDAG &	DAG
	)

static

Definition at line 4560 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::BITCAST, DL, llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::Lo, and llvm::ISD::SRA.

Referenced by splatSplitI64WithVL().

◆ splatSplitI64WithVL()

static SDValue splatSplitI64WithVL	(	const SDLoc &	DL,
		MVT	VT,
		SDValue	Passthru,
		SDValue	Scalar,
		SDValue	VL,
		SelectionDAG &	DAG
	)

static

Definition at line 4608 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::Hi, llvm::Lo, splatPartsI64WithVL(), and llvm::SelectionDAG::SplitScalar().

Referenced by lowerScalarSplat(), and lowerVectorIntrinsicScalars().

◆ SplitStrictFPVectorOp()

static SDValue SplitStrictFPVectorOp	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 7228 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ SplitVectorOp()

static SDValue SplitVectorOp	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 7157 of file RISCVISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ SplitVectorReductionOp()

static SDValue SplitVectorReductionOp	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 7213 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::SplitEVL(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ SplitVPOp()

static SDValue SplitVPOp	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Definition at line 7182 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::ISD::getVPExplicitVectorLengthIdx(), llvm::ISD::isVPOpcode(), llvm::SelectionDAG::SplitEVL(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

◆ STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

◆ transformAddImmMulImm()

static SDValue transformAddImmMulImm	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15495 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), llvm::ISD::MUL, and N.

Referenced by performADDCombine().

◆ transformAddShlImm()

static SDValue transformAddShlImm	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 15273 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), N, and llvm::ISD::SHL.

Referenced by performADDCombine().

◆ translateSetCCForBranch()

static void translateSetCCForBranch	(	const SDLoc &	DL,
		SDValue &	LHS,
		SDValue &	RHS,
		ISD::CondCode &	CC,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

◆ tryDemorganOfBooleanCondition()

static SDValue tryDemorganOfBooleanCondition	(	SDValue	Cond,
		SelectionDAG &	DAG
	)

static

◆ tryFoldSelectIntoOp()

static SDValue tryFoldSelectIntoOp	(	SDNode *	N,
		SelectionDAG &	DAG,
		SDValue	TrueVal,
		SDValue	FalseVal,
		bool	Swapped
	)

static

Definition at line 18683 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNeutralElement(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getValueType(), N, Opc, llvm::ISD::OR, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), and llvm::ISD::XOR.

Referenced by performSELECTCombine().

◆ tryMemPairCombine()

static SDValue tryMemPairCombine	(	SelectionDAG &	DAG,
		LSBaseSDNode *	LSNode1,
		LSBaseSDNode *	LSNode2,
		SDValue	BasePtr,
		uint64_t	Imm
	)

static

◆ tryWidenMaskForShuffle()

static SDValue tryWidenMaskForShuffle	(	SDValue	Op,
		SelectionDAG &	DAG
	)

static

Try to widen element type to get a new mask value for a better permutation sequence.

This doesn't try to inspect the widened mask for profitability; we speculate the widened form is equal or better. This has the effect of reducing mask constant sizes - allowing cheaper materialization sequences

and index sequence sizes - reducing register pressure and materialization cost, at the cost of (possibly) an extra VTYPE toggle.

Definition at line 5694 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::widenShuffleMaskElts().

◆ unpackF64OnRV32DSoftABI()

static SDValue unpackF64OnRV32DSoftABI	(	SelectionDAG &	DAG,
		SDValue	Chain,
		const CCValAssign &	VA,
		const CCValAssign &	HiVA,
		const SDLoc &	DL
	)

static

◆ unpackFromMemLoc()

static SDValue unpackFromMemLoc	(	SelectionDAG &	DAG,
		SDValue	Chain,
		const CCValAssign &	VA,
		const SDLoc &	DL
	)

static

◆ unpackFromRegLoc()

static SDValue unpackFromRegLoc	(	SelectionDAG &	DAG,
		SDValue	Chain,
		const CCValAssign &	VA,
		const SDLoc &	DL,
		const ISD::InputArg &	In,
		const RISCVTargetLowering &	TLI
	)

static

◆ useInversedSetcc()

static SDValue useInversedSetcc	(	SDNode *	N,
		SelectionDAG &	DAG,
		const RISCVSubtarget &	Subtarget
	)

static

Definition at line 18794 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, Cond, DL, llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::isNullConstant(), llvm::APInt::isPowerOf2(), llvm::EVT::isScalarInteger(), llvm::APInt::isSignedIntN(), LHS, N, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, and llvm::ISD::SETNE.

Referenced by performSELECTCombine().

◆ useRVVForFixedLengthVectorVT()

static bool useRVVForFixedLengthVectorVT	(	MVT	VT,
		const RISCVSubtarget &	Subtarget
	)

static

◆ useTpOffset()

static Value * useTpOffset	(	IRBuilderBase &	IRB,
		unsigned	Offset
	)

static

Definition at line 24502 of file RISCVISelLowering.cpp.

References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::BasicBlock::getModule(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.

Referenced by llvm::RISCVTargetLowering::getIRStackGuard().

◆ widenVectorOpsToi8()

static SDValue widenVectorOpsToi8	(	SDValue	N,
		const SDLoc &	DL,
		SelectionDAG &	DAG
	)

static

Definition at line 12012 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementType(), DL, llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), I, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SETNE, llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::ZERO_EXTEND.

Variable Documentation

◆ AllowSplatInVW_W

cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))	(	DEBUG_TYPE "-form-vw-w-with-splat"	,
		cl::Hidden	,
		cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants")	,
		cl::init(false)
	)

static

◆ ExtensionMaxWebSize

cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))	(	DEBUG_TYPE "-ext-max-web-size"	,
		cl::Hidden	,
		cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion")	,
		cl::init(18)
	)

static

Referenced by combineOp_VLToVWOp_VL().

◆ FPImmCost

cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))	(	DEBUG_TYPE "-fpimm-cost"	,
		cl::Hidden	,
		cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value")	,
		cl::init(2)
	)

static

Referenced by llvm::RISCVTargetLowering::isFPImmLegal().

◆ ModeMask32

const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL

Definition at line 14192 of file RISCVISelLowering.cpp.

◆ ModeMask64

const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL

Definition at line 14191 of file RISCVISelLowering.cpp.

◆ NumRepeatedDivisors

cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))	(	DEBUG_TYPE "-fp-repeated-divisors"	,
		cl::Hidden	,
		cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal")	,
		cl::init(2)
	)

static

◆ ReassocShlAddiAdd

cl::opt< bool > ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden, cl::desc("Swap add and addi in cases where the add may " "be combined with a shift"), cl::init(true))	(	"reassoc-shl-addi-add"	,
		cl::Hidden	,
		cl::desc("Swap add and addi in cases where the add may " "be combined with a shift")	,
		cl::init(true)
	)

static

Referenced by combineShlAddIAdd().

Classes

Namespaces

Macros

Functions

Variables

Macro Definition Documentation

◆ CC_VLS_CASE

◆ DEBUG_TYPE

◆ GET_REGISTER_MATCHER

◆ GET_RISCVVIntrinsicsTable_IMPL

◆ OP_CASE

◆ VP_CASE

Function Documentation

◆ combine_CC()

◆ combineAddOfBooleanXor()

◆ combineBinOpOfExtractToReduceTree()

◆ combineBinOpOfZExt()

◆ combineBinOpToReduce()

◆ combineDeMorganOfBoolean()

◆ combineOp_VLToVWOp_VL()

◆ combineOrOfCZERO()

◆ combineSelectAndUse()

◆ combineSelectAndUseCommutative()

◆ combineSelectToBinOp()

◆ combineShlAddIAdd()

◆ combineShlAddIAddImpl()

◆ combineSubOfBoolean()

◆ combineSubShiftToOrcB()

◆ combineToVCPOP()

◆ combineToVWMACC()

◆ combineTruncOfSraSext()

◆ combineTruncSelectToSMaxUSat()

◆ combineTruncToVnclip()

◆ combineVectorMulToSraBitcast()

◆ combineVectorSizedSetCCEquality()

◆ combineVFMADD_VLWithVFNEG_VL()

◆ combineVqdotAccum()

◆ combineVWADDSUBWSelect()

◆ computeGREVOrGORC()

◆ convertFromScalableVector()

◆ convertLocVTToValVT()

◆ convertToScalableVector()

◆ convertValVTToLocVT()

◆ customLegalizeToWOp()

◆ customLegalizeToWOpWithSExt()

◆ emitBuildPairF64Pseudo()

◆ emitFROUND()

◆ EmitLoweredCascadedSelect()

◆ emitQuietFCMP()

◆ emitReadCounterWidePseudo()

◆ emitSelectPseudo()

◆ emitSplitF64Pseudo()

◆ emitVFROUND_NOEXCEPT_MASK()

◆ expandMul()

◆ expandMulToAddOrSubOfShl()

◆ expandMulToNAFSequence()

◆ foldBinOpIntoSelectIfProfitable()

◆ foldConcatVector()

◆ foldReduceOperandViaVQDOT()

◆ foldSelectOfCTTZOrCTLZ()

◆ getAllOnesMask()

◆ getContainerForFixedLengthVector() [1/2]

◆ getContainerForFixedLengthVector() [2/2]

◆ getDefaultScalableVLOps()

◆ getDefaultVLOps() [1/2]

◆ getDefaultVLOps() [2/2]

◆ getDeinterleaveShiftAndTrunc()

◆ getExactInteger()

◆ getIntrinsicForMaskedAtomicRMWBinOp()

◆ getLargeExternalSymbol()

◆ getLargeGlobalAddress()

◆ getMaskTypeFor()

◆ getPACKOpcode()

◆ getPrefTypeAlign()

◆ getQDOTXResultType()

◆ getRISCVVLOp()

◆ getRISCVWOpcode()

◆ getRVVFPReductionOpAndOperands()

◆ getRVVReductionOp()

◆ getSingleShuffleSrc()