#include "Transforms/Vectorize/VPlanTransforms.h"

Static Public Member Functions
template<typename... ArgsTy>
static bool	runPass (bool(*Transform)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
	Helper to run a VPlan transform `Transform` on `VPlan`, forwarding extra arguments to the transform.

template<typename... ArgsTy>
static void	runPass (void(*Fn)(VPlan &, ArgsTy...), VPlan &Plan, typename std::remove_reference< ArgsTy >::type &...Args)
	Helper to run a VPlan transform `Transform` on `VPlan`, forwarding extra arguments to the transform.

static LLVM_ABI_FOR_TEST std::unique_ptr< VPlan >	buildVPlan0 (Loop TheLoop, LoopInfo &LI, Type InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE)
	Create a base VPlan0, serving as the common starting point for all later candidates.

static LLVM_ABI_FOR_TEST void	handleEarlyExits (VPlan &Plan, bool HasUncountableExit)
	Update `Plan` to account for all early exits.

static LLVM_ABI_FOR_TEST void	addMiddleCheck (VPlan &Plan, bool RequiresScalarEpilogueCheck, bool TailFolded)
	If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block.

static void	addMinimumIterationCheck (VPlan &Plan, ElementCount VF, unsigned UF, ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue, bool TailFolded, bool CheckNeededWithTailFolding, Loop OrigLoop, const uint32_t MinItersBypassWeights, DebugLoc DL, ScalarEvolution &SE)

static LLVM_ABI_FOR_TEST void	createLoopRegions (VPlan &Plan)
	Replace loops in `Plan's` flat CFG with VPRegionBlocks, turning `Plan's` flat CFG into a hierarchical CFG.

static void	attachCheckBlock (VPlan &Plan, Value Cond, BasicBlock CheckBlock, bool AddBranchWeights)
	Wrap runtime check block `CheckBlock` in a VPIRBB and `Cond` in a VPValue and connect the block to `Plan`, using the VPValue as branch condition.

static LLVM_ABI_FOR_TEST bool	tryToConvertVPInstructionsToVPRecipes (VPlanPtr &Plan, function_ref< const InductionDescriptor (PHINode )> GetIntOrFpInductionDescriptor, const TargetLibraryInfo &TLI)
	Replaces the VPInstructions in `Plan` with corresponding widen recipes.

static bool	adjustFixedOrderRecurrences (VPlan &Plan, VPBuilder &Builder)
	Try to have all users of fixed-order recurrences appear after the recipe defining their previous value, by either sinking users or hoisting recipes defining their previous value (and its operands).

static bool	handleMaxMinNumReductions (VPlan &Plan)
	Check if `Plan` contains any FMaxNum or FMinNum reductions.

static void	clearReductionWrapFlags (VPlan &Plan)
	Clear NSW/NUW flags from reduction instructions if necessary.

static void	unrollByUF (VPlan &Plan, unsigned UF)
	Explicitly unroll `Plan` by `UF`.

static void	replicateByVF (VPlan &Plan, ElementCount VF)
	Replace each VPReplicateRecipe outside on any replicate region in `Plan` with `VF` single-scalar recipes.

static void	optimizeForVFAndUF (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
	Optimize `Plan` based on `BestVF` and `BestUF`.

static void	optimize (VPlan &Plan)
	Apply VPlan-to-VPlan optimizations to `Plan`, including induction recipe optimizations, dead recipe removal, replicate region optimizations and block merging.

static void	createAndOptimizeReplicateRegions (VPlan &Plan)
	Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask operand.

static void	addActiveLaneMask (VPlan &Plan, bool UseActiveLaneMaskForControlFlow, bool DataAndControlFlowWithoutRuntimeCheck)
	Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe, wide canonical IV, trip-count).

static void	truncateToMinimalBitwidths (VPlan &Plan, const MapVector< Instruction *, uint64_t > &MinBWs)
	Insert truncates and extends for any truncated recipe.

static void	dropPoisonGeneratingRecipes (VPlan &Plan, const std::function< bool(BasicBlock *)> &BlockNeedsPredication)
	Drop poison flags from recipes that may generate a poison value that is used after vectorization, even when their operands are not poison.

static void	addExplicitVectorLength (VPlan &Plan, const std::optional< unsigned > &MaxEVLSafeElements)
	Add a VPEVLBasedIVPHIRecipe and related recipes to `Plan` and replaces all uses except the canonical IV increment of VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.

static void	createInterleaveGroups (VPlan &Plan, const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &InterleaveGroups, VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed)

static void	removeDeadRecipes (VPlan &Plan)
	Remove dead recipes from `Plan`.

static void	handleUncountableEarlyExit (VPBasicBlock EarlyExitingVPBB, VPBasicBlock EarlyExitVPBB, VPlan &Plan, VPBasicBlock HeaderVPBB, VPBasicBlock LatchVPBB)
	Update `Plan` to account for the uncountable early exit from `EarlyExitingVPBB` to `EarlyExitVPBB` by.

static void	dissolveLoopRegions (VPlan &Plan)
	Replace loop regions with explicit CFG.

static void	canonicalizeEVLLoops (VPlan &Plan)
	Transform EVL loops to use variable-length stepping after region dissolution.

static void	convertToConcreteRecipes (VPlan &Plan)
	Lower abstract recipes to concrete ones, that can be codegen'd.

static void	convertToAbstractRecipes (VPlan &Plan, VPCostContext &Ctx, VFRange &Range)
	This function converts initial recipes to the abstract recipes and clamps `Range` based on cost model for following optimizations and cost estimations.

static void	simplifyRecipes (VPlan &Plan)
	Perform instcombine-like simplifications on recipes in `Plan`.

static void	removeBranchOnConst (VPlan &Plan)
	Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their successors.

static void	optimizeInductionExitUsers (VPlan &Plan, DenseMap< VPValue , VPValue > &EndValues, ScalarEvolution &SE)
	If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them precomputed end values instead, possibly taken one step backwards.

static void	materializeBroadcasts (VPlan &Plan)
	Add explicit broadcasts for live-ins and VPValues defined in `Plan's` entry block if they are used as vectors.

static void	materializeConstantVectorTripCount (VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)

static void	materializeVectorTripCount (VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking, bool RequiresScalarEpilogue)
	Materialize vector trip count computations to a set of VPInstructions.

static void	materializeBackedgeTakenCount (VPlan &Plan, VPBasicBlock *VectorPH)
	Materialize the backedge-taken count to be computed explicitly using VPInstructions.

static void	materializeBuildVectors (VPlan &Plan)
	Add explicit Build[Struct]Vector recipes that combine multiple scalar values into single vectors.

static void	materializeVFAndVFxUF (VPlan &Plan, VPBasicBlock *VectorPH, ElementCount VF)
	Materialize VF and VFxUF to be computed explicitly using VPInstructions.

static DenseMap< const SCEV , Value >	expandSCEVs (VPlan &Plan, ScalarEvolution &SE)
	Expand VPExpandSCEVRecipes in `Plan's` entry block.

static void	narrowInterleaveGroups (VPlan &Plan, ElementCount VF, unsigned VectorRegWidth)
	Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups replaced by wide loads and stores processing VF elements, if all transformed interleave groups access the full vector width (checked via \o VectorRegWidth).

static DenseMap< VPBasicBlock , VPValue >	introduceMasksAndLinearize (VPlan &Plan, bool FoldTail)
	Predicate and linearize the control-flow in the only loop region of `Plan`.

static void	addBranchWeightToMiddleTerminator (VPlan &Plan, ElementCount VF, std::optional< unsigned > VScaleForTuning)
	Add branch weight metadata, if the `Plan's` middle block is terminated by a BranchOnCond recipe.

Detailed Description

Definition at line 36 of file VPlanTransforms.h.

Member Function Documentation

◆ addActiveLaneMask()

void VPlanTransforms::addActiveLaneMask	(	VPlan &	Plan,
		bool	UseActiveLaneMaskForControlFlow,
		bool	DataAndControlFlowWithoutRuntimeCheck
	)

static

Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an (active-lane-mask recipe, wide canonical IV, trip-count).

If UseActiveLaneMaskForControlFlow is true, introduce an VPActiveLaneMaskPHIRecipe. If DataAndControlFlowWithoutRuntimeCheck is true, no minimum-iteration runtime check will be created (during skeleton creation) and instead it is handled using active-lane-mask. DataAndControlFlowWithoutRuntimeCheck implies UseActiveLaneMaskForControlFlow.

Definition at line 2127 of file VPlanTransforms.cpp.

References llvm::VPInstruction::ActiveLaneMask, addVPLaneMaskPhiAndUpdateExitBranch(), assert(), B, llvm::DataAndControlFlowWithoutRuntimeCheck, llvm::VPRecipeBase::eraseFromParent(), llvm::find_if(), findHeaderMask(), llvm::VPlan::getCanonicalIV(), llvm::VPBuilder::getToInsertAfter(), llvm::VPlan::getTripCount(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::users().

◆ addBranchWeightToMiddleTerminator()

void VPlanTransforms::addBranchWeightToMiddleTerminator	(	VPlan &	Plan,
		ElementCount	VF,
		std::optional< unsigned >	VScaleForTuning
	)

static

Add branch weight metadata, if the Plan's middle block is terminated by a BranchOnCond recipe.

Definition at line 3782 of file VPlanTransforms.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::MDBuilder::createBranchWeights(), llvm::VPlan::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPlan::getMiddleBlock(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getUF(), and llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ addExplicitVectorLength()

void VPlanTransforms::addExplicitVectorLength	(	VPlan &	Plan,
		const std::optional< unsigned > &	MaxSafeElements
	)

static

Add a VPEVLBasedIVPHIRecipe and related recipes to Plan and replaces all uses except the canonical IV increment of VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.

VPCanonicalIVPHIRecipe is only used to control the loop after this transformation.

VPCanonicalIVPHIRecipe is used only for loop iterations counting after this transformation.

The function uses the following definitions: StartV is the canonical induction start value.

The function adds the following recipes:

vector.ph: ...

vector.body: ... EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ StartV, vector.ph ], [ NextEVLIV, vector.body ] AVL = phi [ trip-count, vector.ph ], [ NextAVL, vector.body ] VPEVL = EXPLICIT-VECTOR-LENGTH AVL ... OpEVL = cast i32 VPEVL to IVSize NextEVLIV = add IVSize OpEVL, EVLPhi NextAVL = sub IVSize nuw AVL, OpEVL ...

If MaxSafeElements is provided, the function adds the following recipes: vector.ph: ...

vector.body: ... EVLPhi = EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI [ StartV, vector.ph ], [ NextEVLIV, vector.body ] AVL = phi [ trip-count, vector.ph ], [ NextAVL, vector.body ] cmp = cmp ult AVL, MaxSafeElements SAFE_AVL = select cmp, AVL, MaxSafeElements VPEVL = EXPLICIT-VECTOR-LENGTH SAFE_AVL ... OpEVL = cast i32 VPEVL to IVSize NextEVLIV = add IVSize OpEVL, EVLPhi NextAVL = sub IVSize nuw AVL, OpEVL ...

Definition at line 2405 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createOverflowingOp(), llvm::VPBuilder::createScalarPhi(), llvm::VPBuilder::createScalarZExtOrTrunc(), llvm::VPBuilder::createSelect(), llvm::VPInstruction::ExplicitVectorLength, llvm::VPlan::getCanonicalIV(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPlan::getContext(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::Type::getInt32Ty(), llvm::VPlan::getOrAddLiveIn(), llvm::VPCanonicalIVPHIRecipe::getScalarType(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorLoopRegion(), llvm::CmpInst::ICMP_ULT, llvm::VPBuilder::setInsertPoint(), llvm::VPlan::setUF(), and transformRecipestoEVLRecipes().

◆ addMiddleCheck()

void VPlanTransforms::addMiddleCheck	(	VPlan &	Plan,
		bool	RequiresScalarEpilogueCheck,
		bool	TailFolded
	)

static

If a check is needed to guard executing the scalar epilogue loop, it will be added to the middle block.

Definition at line 576 of file VPlanConstruction.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPlan::getContext(), llvm::VPlan::getFalse(), llvm::Type::getInt1Ty(), llvm::VPlan::getOrAddLiveIn(), llvm::VPBlockBase::getPredecessors(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getTripCount(), llvm::ConstantInt::getTrue(), llvm::VPlan::getVectorTripCount(), and llvm::CmpInst::ICMP_EQ.

◆ addMinimumIterationCheck()

void VPlanTransforms::addMinimumIterationCheck	(	VPlan &	Plan,
		ElementCount	VF,
		unsigned	UF,
		ElementCount	MinProfitableTripCount,
		bool	RequiresScalarEpilogue,
		bool	TailFolded,
		bool	CheckNeededWithTailFolding,
		Loop *	OrigLoop,
		const uint32_t *	MinItersBypassWeights,
		DebugLoc	DL,
		ScalarEvolution &	SE
	)

static

Definition at line 672 of file VPlanConstruction.cpp.

References llvm::ScalarEvolution::applyLoopGuards(), llvm::VPInstruction::BranchOnCond, llvm::MDBuilder::createBranchWeights(), llvm::VPBuilder::createExpandSCEV(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), DL, llvm::SCEV::FlagNUW, llvm::VPlan::getContext(), llvm::ScalarEvolution::getElementCount(), llvm::VPlan::getEntry(), llvm::VPlan::getFalse(), llvm::CmpInst::getInversePredicate(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getMask(), llvm::VPlan::getOrAddLiveIn(), llvm::vputils::getSCEVExprForVPValue(), llvm::VPlan::getTripCount(), llvm::VPlan::getTrue(), llvm::SCEV::getType(), llvm::ScalarEvolution::getUMaxExpr(), llvm::DebugLoc::getUnknown(), llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, llvm::ScalarEvolution::isKnownPredicate(), and MinItersBypassWeights.

Referenced by llvm::LoopVectorizationPlanner::addMinimumIterationCheck().

◆ adjustFixedOrderRecurrences()

bool VPlanTransforms::adjustFixedOrderRecurrences	(	VPlan &	Plan,
		VPBuilder &	Builder
	)

static

Try to have all users of fixed-order recurrences appear after the recipe defining their previous value, by either sinking users or hoisting recipes defining their previous value (and its operands).

Then introduce FirstOrderRecurrenceSplice VPInstructions to combine the value from the recurrence phis and previous values.

Returns: true if all users of fixed-order recurrences could be re-arranged as needed or false if it is not possible. In the latter case, Plan is not valid.

Definition at line 1736 of file VPlanTransforms.cpp.

References assert(), llvm::VPBuilder::createNaryOp(), llvm::VPInstruction::FirstOrderRecurrenceSplice, llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBasicBlock::getFirstNonPhi(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::VPRecipeBase::getParent(), llvm::VPlan::getVectorLoopRegion(), hoistPreviousBeforeFORUsers(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::VPBasicBlock::phis(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::DominatorTreeBase< NodeT, IsPostDom >::recalculate(), llvm::VPBuilder::setInsertPoint(), and sinkRecurrenceUsersAfterPrevious().

◆ attachCheckBlock()

void VPlanTransforms::attachCheckBlock	(	VPlan &	Plan,
		Value *	Cond,
		BasicBlock *	CheckBlock,
		bool	AddBranchWeights
	)

static

Wrap runtime check block CheckBlock in a VPIRBB and Cond in a VPValue and connect the block to Plan, using the VPValue as branch condition.

Definition at line 638 of file VPlanConstruction.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, CheckBypassWeights, Cond, llvm::VPBlockUtils::connectBlocks(), llvm::MDBuilder::createBranchWeights(), llvm::VPBuilder::createNaryOp(), llvm::VPlan::createVPIRBasicBlock(), llvm::VPlan::getCanonicalIV(), llvm::VPlan::getContext(), getDebugLoc(), llvm::VPBlockBase::getNumPredecessors(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getVectorPreheader(), llvm::VPBlockUtils::insertOnEdge(), and llvm::VPBlockBase::swapSuccessors().

◆ buildVPlan0()

std::unique_ptr< VPlan > VPlanTransforms::buildVPlan0	(	Loop *	TheLoop,
		LoopInfo &	LI,
		Type *	InductionTy,
		DebugLoc	IVDL,
		PredicatedScalarEvolution &	PSE
	)

static

Create a base VPlan0, serving as the common starting point for all later candidates.

It consists of an initial plain CFG loop with loop blocks from TheLoop being directly translated to VPBasicBlocks with VPInstruction corresponding to the input IR.

The created loop is wrapped in an initial skeleton to facilitate vectorization, consisting of a vector pre-header, an exit block for the main vector loop (middle.block) and a new block as preheader of the scalar loop (scalar.ph). See below for an illustration. It also adds a canonical IV and its increment, using InductionTy and IVDL, and creates a VPValue expression for the original trip count.

[ ] <– Plan's entry VPIRBasicBlock, wrapping the original loop's / \ old preheader. Will contain iteration number check and SCEV | | expansions. | | / v | [ ] <– vector loop bypass (may consist of multiple blocks) will be | / | added later. | / v || [ ] <– vector pre header. |/ | | v | [ ] \ <– plain CFG loop wrapping original loop to be vectorized. | [ ]_| | | | v | [ ] <— middle-block with the branch to successors | / | | / | | | v --->[ ] <— scalar preheader (initial a VPBasicBlock, which will be | | replaced later by a VPIRBasicBlock wrapping the scalar | | preheader basic block. | | v <– edge from middle to exit iff epilogue is not required. | [ ] \ | [ ]_| <– old scalar loop to handle remainder (scalar epilogue, | | header wrapped in VPIRBasicBlock). \ | \ v >[ ] <– original loop exit block(s), wrapped in VPIRBasicBlocks.

Definition at line 532 of file VPlanConstruction.cpp.

References addInitialSkeleton().

◆ canonicalizeEVLLoops()

void VPlanTransforms::canonicalizeEVLLoops ( VPlan & Plan )

static

Transform EVL loops to use variable-length stepping after region dissolution.

Once loop regions are replaced with explicit CFG, EVL loops can step with variable vector lengths instead of fixed lengths. This transformation:

Makes EVL-Phi concrete.

Replaces the exit condition from (branch-on-count CanonicalIVInc, VectorTripCount) to (branch-on-cond eq AVLNext, 0)

Definition at line 2464 of file VPlanTransforms.cpp.

References assert(), llvm::VPBasicBlock::begin(), llvm::VPInstruction::BranchOnCond, llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createScalarPhi(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPHeaderPHIRecipe::getBackedgeValue(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPValue::getDefiningRecipe(), llvm::VPlan::getEntry(), llvm::Constant::getNullValue(), llvm::VPlan::getOrAddLiveIn(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getPredecessors(), llvm::VPHeaderPHIRecipe::getStartValue(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::PatternMatch::m_c_Add(), llvm::VPlanPatternMatch::m_EVL(), llvm::PatternMatch::m_Select(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::m_ZExtOrSelf(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ clearReductionWrapFlags()

void VPlanTransforms::clearReductionWrapFlags ( VPlan & Plan )

static

Clear NSW/NUW flags from reduction instructions if necessary.

Definition at line 1784 of file VPlanTransforms.cpp.

References llvm::Add, llvm::AddChainWithSubs, collectUsersRecursively(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::Mul, llvm::VPBasicBlock::phis(), and llvm::Sub.

◆ convertToAbstractRecipes()

void VPlanTransforms::convertToAbstractRecipes	(	VPlan &	Plan,
		VPCostContext &	Ctx,
		VFRange &	Range
	)

static

This function converts initial recipes to the abstract recipes and clamps Range based on cost model for following optimizations and cost estimations.

The converted abstract recipes will lower to concrete recipes before codegen.

Definition at line 3268 of file VPlanTransforms.cpp.

References llvm::VPlan::getVectorLoopRegion(), llvm::make_early_inc_range(), Range, tryToCreateAbstractReductionRecipe(), and llvm::vp_depth_first_deep().

◆ convertToConcreteRecipes()

void VPlanTransforms::convertToConcreteRecipes ( VPlan & Plan )

static

Lower abstract recipes to concrete ones, that can be codegen'd.

Definition at line 2936 of file VPlanTransforms.cpp.

References assert(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createSelect(), llvm::VPBuilder::createWidenCast(), expandVPWidenIntOrFpInduction(), expandVPWidenPointerInduction(), llvm::VPlan::getEntry(), llvm::VPValue::getLiveInIRValue(), I, llvm::VPTypeAnalysis::inferScalarType(), llvm::Type::isFloatingPointTy(), llvm::VPValue::isLiveIn(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::Mul, llvm::VPValue::replaceAllUsesWith(), Select, ToRemove, and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ createAndOptimizeReplicateRegions()

void VPlanTransforms::createAndOptimizeReplicateRegions ( VPlan & Plan )

static

Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask operand.

Optimize the created regions by iteratively sinking scalar operands into the region, followed by merging regions until no improvements are remaining.

Definition at line 447 of file VPlanTransforms.cpp.

References addReplicateRegions(), mergeBlocksIntoPredecessors(), mergeReplicateRegionsIntoSuccessors(), and sinkScalarOperands().

Referenced by optimize().

◆ createInterleaveGroups()

void VPlanTransforms::createInterleaveGroups	(	VPlan &	Plan,
		const SmallPtrSetImpl< const InterleaveGroup< Instruction > * > &	InterleaveGroups,
		VPRecipeBuilder &	RecipeBuilder,
		const bool &	ScalarEpilogueAllowed
	)

static

◆ createLoopRegions()

void VPlanTransforms::createLoopRegions ( VPlan & Plan )

static

Replace loops in Plan's flat CFG with VPRegionBlocks, turning Plan's flat CFG into a hierarchical CFG.

Definition at line 622 of file VPlanConstruction.cpp.

References canonicalHeaderAndLatch(), createLoopRegion(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), llvm::DominatorTreeBase< NodeT, IsPostDom >::recalculate(), llvm::VPBlockBase::setName(), and llvm::vp_post_order_shallow().

◆ dissolveLoopRegions()

void VPlanTransforms::dissolveLoopRegions ( VPlan & Plan )

static

Replace loop regions with explicit CFG.

Definition at line 2924 of file VPlanTransforms.cpp.

References llvm::VPlan::getEntry(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ dropPoisonGeneratingRecipes()

void VPlanTransforms::dropPoisonGeneratingRecipes	(	VPlan &	Plan,
		const std::function< bool(BasicBlock *)> &	BlockNeedsPredication
	)

static

Drop poison flags from recipes that may generate a poison value that is used after vectorization, even when their operands are not poison.

Those recipes meet the following conditions:

Contribute to the address computation of a recipe generating a widen memory load/store (VPWidenMemoryInstructionRecipe or VPInterleaveRecipe).
Such a widen memory load/store has at least one underlying Instruction that is in a basic block that needs predication and after vectorization the generated instruction won't be predicated. Uses BlockNeedsPredication to check if a block needs predicating. TODO: Replace BlockNeedsPredication callback with retrieving info from VPlan directly.

Definition at line 2545 of file VPlanTransforms.cpp.

References llvm::SmallVectorBase< Size_T >::empty(), for(), if(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ expandSCEVs()

DenseMap< const SCEV , Value > VPlanTransforms::expandSCEVs	(	VPlan &	Plan,
		ScalarEvolution &	SE
	)

static

Expand VPExpandSCEVRecipes in Plan's entry block.

Each VPExpandSCEVRecipe is replaced with a live-in wrapping the expanded IR value. A mapping from SCEV expressions to their expanded IR value is returned.

Definition at line 3512 of file VPlanTransforms.cpp.

References assert(), llvm::VPIRInstruction::create(), DL, llvm::drop_end(), llvm::SCEVExpander::expandCodeFor(), llvm::ScalarEvolution::getDataLayout(), llvm::VPlan::getEntry(), llvm::VPlan::getOrAddLiveIn(), llvm::BasicBlock::getTerminator(), llvm::VPlan::getTripCount(), llvm::SCEV::getType(), I, llvm::VPRecipeBase::insertBefore(), llvm::make_early_inc_range(), llvm::none_of(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPlan::resetTripCount().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ handleEarlyExits()

void VPlanTransforms::handleEarlyExits	(	VPlan &	Plan,
		bool	HasUncountableExit
	)

static

Update Plan to account for all early exits.

Definition at line 540 of file VPlanConstruction.cpp.

References assert(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPlan::getExitBlocks(), llvm::VPBlockBase::getPredecessors(), llvm::VPlan::getScalarHeader(), llvm::VPBlockBase::getSinglePredecessor(), handleUncountableEarlyExit(), and llvm::to_vector().

◆ handleMaxMinNumReductions()

bool VPlanTransforms::handleMaxMinNumReductions ( VPlan & Plan )

static

Check if Plan contains any FMaxNum or FMinNum reductions.

If they do, try to update the vector loop to exit early if any input is NaN and resume executing in the scalar loop to handle the NaNs there. Return false if this attempt was unsuccessful.

Check if the vector loop of Plan can early exit and restart execution of last vector iteration in the scalar loop. This requires all recipes up to early exit point be side-effect free as they are re-executed. Currently we check that the loop is free of any recipe that may write to memory. Expected to operate on an early VPlan w/o nested regions.

Definition at line 756 of file VPlanConstruction.cpp.

References llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::BranchOnCount, llvm::VPBuilder::createAnd(), llvm::VPBuilder::createFCmp(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createNot(), llvm::VPBuilder::createSelect(), llvm::dbgs(), llvm::VPRecipeBase::eraseFromParent(), llvm::CmpInst::FCMP_UNO, llvm::FMaxNum, llvm::FMinNum, llvm::VPlan::getCanonicalIV(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getExitingBasicBlock(), llvm::VPBuilder::getInsertPoint(), llvm::VPlan::getMiddleBlock(), llvm::VPUser::getOperand(), llvm::VPReductionPHIRecipe::getRecurrenceKind(), llvm::VPlan::getScalarPreheader(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::hasScalarTail(), llvm::CmpInst::ICMP_EQ, LLVM_DEBUG, llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPRecipeBase::moveAfter(), llvm::VPBasicBlock::phis(), llvm::VPBuilder::setInsertPoint(), llvm::VPUser::setOperand(), llvm::VPValue::users(), and llvm::vp_depth_first_shallow().

◆ handleUncountableEarlyExit()

void VPlanTransforms::handleUncountableEarlyExit	(	VPBasicBlock *	EarlyExitingVPBB,
		VPBasicBlock *	EarlyExitVPBB,
		VPlan &	Plan,
		VPBasicBlock *	HeaderVPBB,
		VPBasicBlock *	LatchVPBB
	)

static

Update Plan to account for the uncountable early exit from EarlyExitingVPBB to EarlyExitVPBB by.

updating the condition exiting the loop via the latch to include the early exit condition,
splitting the original middle block to branch to the early exit block conditionally - according to the early exit condition.

Definition at line 3016 of file VPlanTransforms.cpp.

References llvm::VPInstruction::AnyOf, assert(), llvm::VPInstruction::BranchOnCond, llvm::VPInstruction::BranchOnCount, llvm::VPBlockUtils::connectBlocks(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createNot(), llvm::VPlan::createVPBasicBlock(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPInstruction::ExtractLane, llvm::VPInstruction::FirstActiveLane, llvm::VPBlockBase::getNumPredecessors(), llvm::VPUser::getOperand(), llvm::VPBlockBase::getPredecessors(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPBlockBase::getSuccessors(), llvm::VPBasicBlock::getTerminator(), llvm::CmpInst::ICMP_EQ, llvm::VPBlockUtils::insertOnEdge(), llvm::VPValue::isLiveIn(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), llvm::VPUser::setOperand(), and llvm::VPBlockBase::swapSuccessors().

Referenced by handleEarlyExits().

◆ introduceMasksAndLinearize()

DenseMap< VPBasicBlock , VPValue > VPlanTransforms::introduceMasksAndLinearize	(	VPlan &	Plan,
		bool	FoldTail
	)

static

Predicate and linearize the control-flow in the only loop region of Plan.

If FoldTail is true, create a mask guarding the loop header, otherwise use all-true for the header mask. Masks for blocks are added to a block-to-mask map which is returned in order to be used later for wide recipe construction. This argument is temporary and will be removed in the future.

Definition at line 261 of file VPlanPredicator.cpp.

References llvm::VPBlockUtils::connectBlocks(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPBlockBase::getSuccessors(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorLoopRegion(), Predicator, and llvm::to_vector().

◆ materializeBackedgeTakenCount()

void VPlanTransforms::materializeBackedgeTakenCount	(	VPlan &	Plan,
		VPBasicBlock *	VectorPH
	)

static

Materialize the backedge-taken count to be computed explicitly using VPInstructions.

Definition at line 3355 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::begin(), llvm::VPBuilder::createNaryOp(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getOrCreateBackedgeTakenCount(), llvm::VPlan::getTripCount(), llvm::VPTypeAnalysis::inferScalarType(), and llvm::VPValue::replaceAllUsesWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeBroadcasts()

void VPlanTransforms::materializeBroadcasts ( VPlan & Plan )

static

Add explicit broadcasts for live-ins and VPValues defined in Plan's entry block if they are used as vectors.

Definition at line 3279 of file VPlanTransforms.cpp.

References llvm::append_range(), assert(), llvm::VPBasicBlock::begin(), llvm::VPInstruction::Broadcast, llvm::VPBuilder::createNaryOp(), llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), llvm::VPBasicBlock::end(), llvm::VPlan::getEntry(), llvm::VPlan::getLiveIns(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrCreateBackedgeTakenCount(), getParent(), llvm::VPlan::getVectorPreheader(), llvm::VPlan::hasScalarVFOnly(), Idx, llvm::vputils::onlyScalarValuesUsed(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::DominatorTreeBase< NodeT, IsPostDom >::recalculate(), llvm::VPValue::replaceUsesWithIf(), and llvm::Value::users().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeBuildVectors()

void VPlanTransforms::materializeBuildVectors ( VPlan & Plan )

static

Add explicit Build[Struct]Vector recipes that combine multiple scalar values into single vectors.

Definition at line 3370 of file VPlanTransforms.cpp.

References llvm::VPInstruction::BuildStructVector, llvm::VPInstruction::BuildVector, llvm::VPlan::getEntry(), llvm::VPRegionBlock::getEntry(), llvm::VPBlockBase::getParent(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::hasScalarVFOnly(), llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertAfter(), llvm::Type::isStructTy(), llvm::make_early_inc_range(), llvm::none_of(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeConstantVectorTripCount()

void VPlanTransforms::materializeConstantVectorTripCount	(	VPlan &	Plan,
		ElementCount	BestVF,
		unsigned	BestUF,
		PredicatedScalarEvolution &	PSE
	)

static

◆ materializeVectorTripCount()

void VPlanTransforms::materializeVectorTripCount	(	VPlan &	Plan,
		VPBasicBlock *	VectorPHVPBB,
		bool	TailByMasking,
		bool	RequiresScalarEpilogue
	)

static

Materialize vector trip count computations to a set of VPInstructions.

Definition at line 3416 of file VPlanTransforms.cpp.

References assert(), llvm::VPBasicBlock::begin(), llvm::VPBuilder::createICmp(), llvm::VPBuilder::createNaryOp(), llvm::VPBuilder::createSelect(), llvm::DebugLoc::getCompilerGenerated(), llvm::VPValue::getLiveInIRValue(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getTripCount(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::CmpInst::ICMP_EQ, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPValue::isLiveIn(), and llvm::VPValue::replaceAllUsesWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ materializeVFAndVFxUF()

void VPlanTransforms::materializeVFAndVFxUF	(	VPlan &	Plan,
		VPBasicBlock *	VectorPH,
		ElementCount	VF
	)

static

Materialize VF and VFxUF to be computed explicitly using VPInstructions.

Definition at line 3477 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::begin(), llvm::VPInstruction::Broadcast, llvm::VPBuilder::createElementCount(), llvm::VPBuilder::createNaryOp(), llvm::VPValue::getNumUsers(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getTripCount(), llvm::VPlan::getUF(), llvm::VPlan::getVF(), llvm::VPlan::getVFxUF(), llvm::VPTypeAnalysis::inferScalarType(), llvm::vputils::onlyScalarValuesUsed(), llvm::VPValue::replaceAllUsesWith(), and llvm::VPValue::replaceUsesWithIf().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ narrowInterleaveGroups()

void VPlanTransforms::narrowInterleaveGroups	(	VPlan &	Plan,
		ElementCount	VF,
		unsigned	VectorRegWidth
	)

static

Try to convert a plan with interleave groups with VF elements to a plan with the interleave groups replaced by wide loads and stores processing VF elements, if all transformed interleave groups access the full vector width (checked via \o VectorRegWidth).

This effectively is a very simple form of loop-aware SLP, where we use interleave groups to identify candidates.

Definition at line 3615 of file VPlanTransforms.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::SmallVectorBase< Size_T >::empty(), llvm::enumerate(), llvm::VPlan::getCanonicalIV(), llvm::VPValue::getDefiningRecipe(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::VPlan::getOrAddLiveIn(), llvm::VPlan::getUF(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVF(), I, Idx, isAlreadyNarrow(), isConsecutiveInterleaveGroup(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isScalable(), llvm::VPlanPatternMatch::m_BranchOnCount(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), N, llvm::none_of(), llvm::vputils::onlyFirstLaneUsed(), P, llvm::SmallVectorTemplateBase< T, bool >::push_back(), removeDeadRecipes(), and llvm::VPValue::replaceAllUsesWith().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ optimize()

void VPlanTransforms::optimize ( VPlan & Plan )

static

Apply VPlan-to-VPlan optimizations to Plan, including induction recipe optimizations, dead recipe removal, replicate region optimizations and block merging.

Definition at line 1958 of file VPlanTransforms.cpp.

References createAndOptimizeReplicateRegions(), legalizeAndOptimizeInductions(), licm, mergeBlocksIntoPredecessors(), narrowToSingleScalarRecipes(), removeBranchOnConst(), removeDeadRecipes(), removeRedundantCanonicalIVs(), removeRedundantExpandSCEVRecipes(), removeRedundantInductionCasts(), runPass(), simplifyBlends(), and simplifyRecipes().

Referenced by llvm::LoopVectorizationPlanner::buildVPlans().

◆ optimizeForVFAndUF()

void VPlanTransforms::optimizeForVFAndUF	(	VPlan &	Plan,
		ElementCount	BestVF,
		unsigned	BestUF,
		PredicatedScalarEvolution &	PSE
	)

static

Optimize Plan based on BestVF and BestUF.

This may restrict the resulting plan to BestVF and BestUF.

Definition at line 1562 of file VPlanTransforms.cpp.

References assert(), llvm::VPlan::getUF(), llvm::VPlan::hasUF(), llvm::VPlan::hasVF(), optimizeVectorInductionWidthForTCAndVFUF(), llvm::VPlan::setVF(), and simplifyBranchConditionForVFAndUF().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ optimizeInductionExitUsers()

void VPlanTransforms::optimizeInductionExitUsers	(	VPlan &	Plan,
		DenseMap< VPValue , VPValue > &	EndValues,
		ScalarEvolution &	SE
	)

static

If there's a single exit block, optimize its phi recipes that use exiting IV values by feeding them precomputed end values instead, possibly taken one step backwards.

Definition at line 877 of file VPlanTransforms.cpp.

References llvm::enumerate(), llvm::VPlan::getExitBlocks(), llvm::VPlan::getMiddleBlock(), Idx, optimizeEarlyExitInductionUser(), and optimizeLatchExitInductionUser().

◆ removeBranchOnConst()

void VPlanTransforms::removeBranchOnConst ( VPlan & Plan )

static

Remove BranchOnCond recipes with true or false conditions together with removing dead edges to their successors.

Definition at line 1926 of file VPlanTransforms.cpp.

References assert(), Cond, llvm::count(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPlan::getEntry(), llvm::VPBlockBase::getPredecessors(), llvm::VPlanPatternMatch::m_BranchOnCond(), llvm::VPlanPatternMatch::m_False(), llvm::VPlanPatternMatch::m_True(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), llvm::VPBasicBlock::phis(), and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), and optimize().

◆ removeDeadRecipes()

void VPlanTransforms::removeDeadRecipes ( VPlan & Plan )

static

Remove dead recipes from Plan.

Definition at line 548 of file VPlanTransforms.cpp.

References llvm::VPlan::getEntry(), isDeadRecipe(), llvm::make_early_inc_range(), llvm::reverse(), and llvm::vp_post_order_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), narrowInterleaveGroups(), optimize(), preparePlanForMainVectorLoop(), and unrollByUF().

◆ replicateByVF()

void VPlanTransforms::replicateByVF	(	VPlan &	Plan,
		ElementCount	VF
	)

static

Replace each VPReplicateRecipe outside on any replicate region in Plan with VF single-scalar recipes.

TODO: Also replicate VPReplicateRecipes inside replicate regions, thereby dissolving the latter.

Create single-scalar version of RepR for all lanes.

Users that only demand the first lane can use the definition for lane 0.

Definition at line 512 of file VPlanUnroll.cpp.

References assert(), llvm::VPInstruction::BuildStructVector, llvm::VPInstruction::BuildVector, cloneForLane(), llvm::drop_begin(), llvm::VPRecipeBase::eraseFromParent(), llvm::IntegerType::get(), llvm::BasicBlock::getContext(), llvm::VPlan::getEntry(), llvm::VPRegionBlock::getEntry(), llvm::VPIRBasicBlock::getIRBasicBlock(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::VPLane::getLastLaneForVF(), llvm::VPlan::getScalarHeader(), llvm::VPlan::getVectorLoopRegion(), I, llvm::vputils::isSingleScalar(), llvm::make_early_inc_range(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reverse(), llvm::to_vector(), ToRemove, and llvm::vp_depth_first_shallow().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

◆ runPass() [1/2]

template<typename... ArgsTy>

static bool llvm::VPlanTransforms::runPass	(	bool(*)(VPlan &, ArgsTy...)	Transform,
		VPlan &	Plan,
		typename std::remove_reference< ArgsTy >::type &...	Args
	)

inlinestatic

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Returns the boolean returned by the transform.

Definition at line 40 of file VPlanTransforms.h.

References llvm::VerifyEachVPlan, and llvm::verifyVPlanIsValid().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), optimize(), preparePlanForMainVectorLoop(), and llvm::LoopVectorizePass::processLoop().

◆ runPass() [2/2]

template<typename... ArgsTy>

static void llvm::VPlanTransforms::runPass	(	void(*)(VPlan &, ArgsTy...)	Fn,
		VPlan &	Plan,
		typename std::remove_reference< ArgsTy >::type &...	Args
	)

inlinestatic

Helper to run a VPlan transform Transform on VPlan, forwarding extra arguments to the transform.

Definition at line 50 of file VPlanTransforms.h.

References llvm::VerifyEachVPlan, and llvm::verifyVPlanIsValid().

◆ simplifyRecipes()

void VPlanTransforms::simplifyRecipes ( VPlan & Plan )

static

Perform instcombine-like simplifications on recipes in Plan.

Definition at line 1250 of file VPlanTransforms.cpp.

References llvm::VPlan::getEntry(), llvm::make_early_inc_range(), and simplifyRecipe().

Referenced by llvm::LoopVectorizationPlanner::executePlan(), optimize(), and simplifyBranchConditionForVFAndUF().

◆ truncateToMinimalBitwidths()

void VPlanTransforms::truncateToMinimalBitwidths	(	VPlan &	Plan,
		const MapVector< Instruction *, uint64_t > &	MinBWs
	)

static

Insert truncates and extends for any truncated recipe.

Redundant casts will be folded later.

Definition at line 1836 of file VPlanTransforms.cpp.

References llvm::VPBasicBlock::appendRecipe(), assert(), llvm::IntegerType::get(), llvm::VPlan::getContext(), llvm::Type::getScalarSizeInBits(), llvm::VPValue::getUnderlyingValue(), llvm::VPlan::getVectorLoopRegion(), llvm::VPlan::getVectorPreheader(), Idx, llvm::VPTypeAnalysis::inferScalarType(), llvm::VPRecipeBase::insertBefore(), llvm::isa(), llvm::Type::isIntegerTy(), llvm::MapVector< KeyT, ValueT, MapType, VectorType >::lookup(), llvm::PatternMatch::m_ICmp(), llvm::VPlanPatternMatch::m_VPValue(), llvm::make_early_inc_range(), llvm::PatternMatch::match(), llvm::VPValue::replaceAllUsesWith(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::try_emplace(), and llvm::vp_depth_first_deep().

◆ tryToConvertVPInstructionsToVPRecipes()

bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes	(	VPlanPtr &	Plan,
		function_ref< const InductionDescriptor (PHINode )>	GetIntOrFpInductionDescriptor,
		const TargetLibraryInfo &	TLI
	)

static

Replaces the VPInstructions in Plan with corresponding widen recipes.

Returns false if any VPInstructions could not be converted to a wide recipe if needed.

Definition at line 42 of file VPlanTransforms.cpp.

References llvm::VPUser::addOperand(), assert(), GEP, llvm::VPDef::getNumDefinedValues(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::VPValue::getUnderlyingValue(), llvm::getVectorIntrinsicIDForCall(), llvm::VPDef::getVPSingleValue(), II, llvm::VPRecipeBase::insertBefore(), llvm::make_early_inc_range(), llvm::make_range(), llvm::Intrinsic::not_intrinsic, llvm::User::operands(), and llvm::VPValue::replaceAllUsesWith().

◆ unrollByUF()

void VPlanTransforms::unrollByUF	(	VPlan &	Plan,
		unsigned	UF
	)

static

Explicitly unroll Plan by UF.

Definition at line 410 of file VPlanUnroll.cpp.

References assert(), llvm::VPInstruction::CanonicalIVIncrementForPart, Cleanup, llvm::VPlan::getEntry(), llvm::VPBlockBase::getEntryBasicBlock(), llvm::VPlan::getVectorLoopRegion(), H, llvm::make_early_inc_range(), llvm::make_scope_exit(), llvm::VPBasicBlock::phis(), removeDeadRecipes(), llvm::VPlan::setUF(), and llvm::vp_depth_first_deep().

Referenced by llvm::LoopVectorizationPlanner::executePlan().

The documentation for this struct was generated from the following files:

lib/Transforms/Vectorize/VPlanTransforms.h
lib/Transforms/Vectorize/VPlanConstruction.cpp
lib/Transforms/Vectorize/VPlanPredicator.cpp
lib/Transforms/Vectorize/VPlanTransforms.cpp
lib/Transforms/Vectorize/VPlanUnroll.cpp

Static Public Member Functions

Detailed Description

Member Function Documentation

◆ addActiveLaneMask()

◆ addBranchWeightToMiddleTerminator()

◆ addExplicitVectorLength()

◆ addMiddleCheck()

◆ addMinimumIterationCheck()

◆ adjustFixedOrderRecurrences()

◆ attachCheckBlock()

◆ buildVPlan0()

◆ canonicalizeEVLLoops()

◆ clearReductionWrapFlags()

◆ convertToAbstractRecipes()

◆ convertToConcreteRecipes()

◆ createAndOptimizeReplicateRegions()

◆ createInterleaveGroups()

◆ createLoopRegions()

◆ dissolveLoopRegions()

◆ dropPoisonGeneratingRecipes()

◆ expandSCEVs()

◆ handleEarlyExits()

◆ handleMaxMinNumReductions()

◆ handleUncountableEarlyExit()

◆ introduceMasksAndLinearize()

◆ materializeBackedgeTakenCount()

◆ materializeBroadcasts()

◆ materializeBuildVectors()

◆ materializeConstantVectorTripCount()

◆ materializeVectorTripCount()

◆ materializeVFAndVFxUF()

◆ narrowInterleaveGroups()

◆ optimize()

◆ optimizeForVFAndUF()

◆ optimizeInductionExitUsers()

◆ removeBranchOnConst()

◆ removeDeadRecipes()

◆ replicateByVF()

◆ runPass() [1/2]

◆ runPass() [2/2]

◆ simplifyRecipes()

◆ truncateToMinimalBitwidths()

◆ tryToConvertVPInstructionsToVPRecipes()

◆ unrollByUF()