}
/// A helper function that returns the alignment of load or store instruction.
-inline MaybeAlign getLoadStoreAlignment(Value *I) {
+inline Align getLoadStoreAlignment(Value *I) {
assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
"Expected Load or Store instruction");
if (auto *LI = dyn_cast<LoadInst>(I))
- return MaybeAlign(LI->getAlignment());
- return MaybeAlign(cast<StoreInst>(I)->getAlignment());
+ return LI->getAlign();
+ return cast<StoreInst>(I)->getAlign();
}
/// A helper function that returns the address space of the pointer operand of
APInt EltSize(DL.getIndexTypeSizeInBits(Ptr->getType()),
DL.getTypeStoreSize(LI->getType()));
- const Align Alignment = DL.getValueOrABITypeAlignment(
- MaybeAlign(LI->getAlignment()), LI->getType());
+ const Align Alignment = LI->getAlign();
Instruction *HeaderFirstNonPHI = L->getHeader()->getFirstNonPHI();
const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
PointerType *PtrTy = cast<PointerType>(Ptr->getType());
uint64_t Size = DL.getTypeAllocSize(PtrTy->getElementType());
-
- // An alignment of 0 means target ABI alignment.
- MaybeAlign Alignment = MaybeAlign(getLoadStoreAlignment(&I));
- if (!Alignment)
- Alignment = Align(DL.getABITypeAlignment(PtrTy->getElementType()));
-
- AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size, *Alignment);
+ AccessStrideInfo[&I] = StrideDescriptor(Stride, Scev, Size,
+ getLoadStoreAlignment(&I));
}
}
if (const StoreInst *SI = dyn_cast<StoreInst>(&I))
return SI->getAlign();
if (const LoadInst *LI = dyn_cast<LoadInst>(&I)) {
- return DL->getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
+ return LI->getAlign();
}
if (const AtomicCmpXchgInst *AI = dyn_cast<AtomicCmpXchgInst>(&I)) {
// TODO(PR27168): This instruction has no alignment attribute, but unlike
SDValue Ptr = getValue(SV);
Type *Ty = I.getType();
- Align Alignment = DL->getValueOrABITypeAlignment(I.getAlign(), Ty);
+ Align Alignment = I.getAlign();
AAMDNodes AAInfo;
I.getAAMetadata(AAInfo);
SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot();
SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
SDLoc dl = getCurSDLoc();
- Align Alignment =
- DL->getValueOrABITypeAlignment(I.getAlign(), SrcV->getType());
+ Align Alignment = I.getAlign();
AAMDNodes AAInfo;
I.getAAMetadata(AAInfo);
const X86InstrInfo &XII = (const X86InstrInfo &)TII;
unsigned Size = DL.getTypeAllocSize(LI->getType());
- Align Alignment =
- DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
SmallVector<MachineOperand, 8> AddrOps;
AM.getFullAddress(AddrOps);
MachineInstr *Result = XII.foldMemoryOperandImpl(
- *FuncInfo.MF, *MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, Alignment,
+ *FuncInfo.MF, *MI, OpNo, AddrOps, FuncInfo.InsertPt, Size, LI->getAlign(),
/*AllowCommute=*/true);
if (!Result)
return false;
if (Size == 0)
return;
- const Align Alignement =
- ClPreserveAlignment ? DL.getValueOrABITypeAlignment(
- SI.getAlign(), SI.getValueOperand()->getType())
- : Align(1);
+ const Align Alignment = ClPreserveAlignment ? SI.getAlign() : Align(1);
Value* Shadow = DFSF.getShadow(SI.getValueOperand());
if (ClCombinePointerLabelsOnStore) {
Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
Shadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
}
- DFSF.storeShadow(SI.getPointerOperand(), Size, Alignement, Shadow, &SI);
+ DFSF.storeShadow(SI.getPointerOperand(), Size, Alignment, Shadow, &SI);
if (ClEventCallbacks) {
IRBuilder<> IRB(&SI);
IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, Shadow);
WorkList.push_back(K);
}
- const DataLayout &DL = SE->getDataLayout();
while (!WorkList.empty()) {
Instruction *J = WorkList.pop_back_val();
if (LoadInst *LI = dyn_cast<LoadInst>(J)) {
Align NewAlignment = getNewAlignment(AASCEV, AlignSCEV, OffSCEV,
LI->getPointerOperand(), SE);
- Align OldAlignment =
- DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
- if (NewAlignment > OldAlignment) {
+ if (NewAlignment > LI->getAlign()) {
LI->setAlignment(NewAlignment);
++NumLoadAlignChanged;
}
} else if (StoreInst *SI = dyn_cast<StoreInst>(J)) {
Align NewAlignment = getNewAlignment(AASCEV, AlignSCEV, OffSCEV,
SI->getPointerOperand(), SE);
- Align OldAlignment = DL.getValueOrABITypeAlignment(
- SI->getAlign(), SI->getOperand(0)->getType());
- if (NewAlignment > OldAlignment) {
+ if (NewAlignment > SI->getAlign()) {
SI->setAlignment(NewAlignment);
++NumStoreAlignChanged;
}
return TheStores.size() > NumPointerStores+NumByteStores;
}
-
-static Align findStoreAlignment(const DataLayout &DL, const StoreInst *SI) {
- return DL.getValueOrABITypeAlignment(SI->getAlign(),
- SI->getOperand(0)->getType());
-}
-
-static Align findLoadAlignment(const DataLayout &DL, const LoadInst *LI) {
- return DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
-}
-
-static Align findCommonAlignment(const DataLayout &DL, const StoreInst *SI,
- const LoadInst *LI) {
- Align StoreAlign = findStoreAlignment(DL, SI);
- Align LoadAlign = findLoadAlignment(DL, LI);
- return commonAlignment(StoreAlign, LoadAlign);
-}
-
namespace {
class MemsetRanges {
int64_t StoreSize = DL.getTypeStoreSize(SI->getOperand(0)->getType());
addRange(OffsetFromFirst, StoreSize, SI->getPointerOperand(),
- findStoreAlignment(DL, SI).value(), SI);
+ SI->getAlign().value(), SI);
}
void addMemSet(int64_t OffsetFromFirst, MemSetInst *MSI) {
Instruction *M;
if (UseMemMove)
M = Builder.CreateMemMove(
- SI->getPointerOperand(), findStoreAlignment(DL, SI),
- LI->getPointerOperand(), findLoadAlignment(DL, LI), Size);
+ SI->getPointerOperand(), SI->getAlign(),
+ LI->getPointerOperand(), LI->getAlign(), Size);
else
M = Builder.CreateMemCpy(
- SI->getPointerOperand(), findStoreAlignment(DL, SI),
- LI->getPointerOperand(), findLoadAlignment(DL, LI), Size);
+ SI->getPointerOperand(), SI->getAlign(),
+ LI->getPointerOperand(), LI->getAlign(), Size);
LLVM_DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI << " => "
<< *M << "\n");
LI, SI->getPointerOperand()->stripPointerCasts(),
LI->getPointerOperand()->stripPointerCasts(),
DL.getTypeStoreSize(SI->getOperand(0)->getType()),
- findCommonAlignment(DL, SI, LI), C);
+ commonAlignment(SI->getAlign(), LI->getAlign()), C);
if (changed) {
MD->removeInstruction(SI);
SI->eraseFromParent();
auto *T = V->getType();
if (T->isAggregateType()) {
uint64_t Size = DL.getTypeStoreSize(T);
- const Align MA =
- DL.getValueOrABITypeAlignment(MaybeAlign(SI->getAlignment()), T);
IRBuilder<> Builder(SI);
- auto *M =
- Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size, MA);
+ auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size,
+ SI->getAlign());
LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
Type *LoadTy = SomeLoad->getType();
- const DataLayout &DL = PN.getModule()->getDataLayout();
IRBuilderTy PHIBuilder(&PN);
PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(),
PN.getName() + ".sroa.speculated");
// matter which one we get and if any differ.
AAMDNodes AATags;
SomeLoad->getAAMetadata(AATags);
- Align Alignment =
- DL.getValueOrABITypeAlignment(SomeLoad->getAlign(), SomeLoad->getType());
+ Align Alignment = SomeLoad->getAlign();
// Rewrite all loads of the PN to use the new PHI.
while (!PN.use_empty()) {
Instruction *TI = Pred->getTerminator();
IRBuilderTy PredBuilder(TI);
- LoadInst *Load = PredBuilder.CreateLoad(
- LoadTy, InVal,
+ LoadInst *Load = PredBuilder.CreateAlignedLoad(
+ LoadTy, InVal, Alignment,
(PN.getName() + ".sroa.speculate.load." + Pred->getName()));
++NumLoadsSpeculated;
- Load->setAlignment(Alignment);
if (AATags)
Load->setAAMetadata(AATags);
NewPN->addIncoming(Load, Pred);
}
/// Compute the adjusted alignment for a load or store from an offset.
-static Align getAdjustedAlignment(Instruction *I, uint64_t Offset,
- const DataLayout &DL) {
- MaybeAlign Alignment;
- Type *Ty;
- if (auto *LI = dyn_cast<LoadInst>(I)) {
- Alignment = MaybeAlign(LI->getAlignment());
- Ty = LI->getType();
- } else if (auto *SI = dyn_cast<StoreInst>(I)) {
- Alignment = MaybeAlign(SI->getAlignment());
- Ty = SI->getValueOperand()->getType();
- } else {
- llvm_unreachable("Only loads and stores are allowed!");
- }
- return commonAlignment(DL.getValueOrABITypeAlignment(Alignment, Ty), Offset);
+static Align getAdjustedAlignment(Instruction *I, uint64_t Offset) {
+ return commonAlignment(getLoadStoreAlignment(I), Offset);
}
/// Test whether we can convert a value from the old to the new type.
/// You can optionally pass a type to this routine and if that type's ABI
/// alignment is itself suitable, this will return zero.
Align getSliceAlign() {
- Align NewAIAlign = DL.getValueOrABITypeAlignment(
- MaybeAlign(NewAI.getAlignment()), NewAI.getAllocatedType());
- return commonAlignment(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset);
+ return commonAlignment(NewAI.getAlign(),
+ NewBeginOffset - NewAllocaBeginOffset);
}
unsigned getIndex(uint64_t Offset) {
Instruction *I = Uses.pop_back_val();
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
- Align LoadAlign =
- DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
- LI->setAlignment(std::min(LoadAlign, getSliceAlign()));
+ LI->setAlignment(std::min(LI->getAlign(), getSliceAlign()));
continue;
}
if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
- Value *Op = SI->getOperand(0);
- Align StoreAlign = DL.getValueOrABITypeAlignment(
- MaybeAlign(SI->getAlignment()), Op->getType());
- SI->setAlignment(std::min(StoreAlign, getSliceAlign()));
- continue;
+ SI->setAlignment(std::min(SI->getAlign(), getSliceAlign()));
+ continue;
}
assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
AAMDNodes AATags;
LI.getAAMetadata(AATags);
LoadOpSplitter Splitter(&LI, *U, LI.getType(), AATags,
- getAdjustedAlignment(&LI, 0, DL), DL);
+ getAdjustedAlignment(&LI, 0), DL);
Value *V = UndefValue::get(LI.getType());
Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca");
LI.replaceAllUsesWith(V);
AAMDNodes AATags;
SI.getAAMetadata(AATags);
StoreOpSplitter Splitter(&SI, *U, V->getType(), AATags,
- getAdjustedAlignment(&SI, 0, DL), DL);
+ getAdjustedAlignment(&SI, 0), DL);
Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
SI.eraseFromParent();
return true;
getAdjustedPtr(IRB, DL, BasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
PartPtrTy, BasePtr->getName() + "."),
- getAdjustedAlignment(LI, PartOffset, DL),
+ getAdjustedAlignment(LI, PartOffset),
/*IsVolatile*/ false, LI->getName());
PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
getAdjustedPtr(IRB, DL, StoreBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
PartPtrTy, StoreBasePtr->getName() + "."),
- getAdjustedAlignment(SI, PartOffset, DL),
+ getAdjustedAlignment(SI, PartOffset),
/*IsVolatile*/ false);
PStore->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
getAdjustedPtr(IRB, DL, LoadBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
LoadPartPtrTy, LoadBasePtr->getName() + "."),
- getAdjustedAlignment(LI, PartOffset, DL),
+ getAdjustedAlignment(LI, PartOffset),
/*IsVolatile*/ false, LI->getName());
}
getAdjustedPtr(IRB, DL, StoreBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
StorePartPtrTy, StoreBasePtr->getName() + "."),
- getAdjustedAlignment(SI, PartOffset, DL),
+ getAdjustedAlignment(SI, PartOffset),
/*IsVolatile*/ false);
// Now build a new slice for the alloca.
// FIXME: We might want to defer PHI speculation until after here.
// FIXME: return nullptr;
} else {
- // If alignment is unspecified we fallback on the one required by the ABI
- // for this type. We also make sure the alignment is compatible with
- // P.beginOffset().
- const Align Alignment = commonAlignment(
- DL.getValueOrABITypeAlignment(MaybeAlign(AI.getAlignment()),
- AI.getAllocatedType()),
- P.beginOffset());
+ // Make sure the alignment is compatible with P.beginOffset().
+ const Align Alignment = commonAlignment(AI.getAlign(), P.beginOffset());
// If we will get at least this much alignment from the type alone, leave
// the alloca's alignment unconstrained.
const bool IsUnconstrained = Alignment <= DL.getABITypeAlignment(SliceTy);
private:
unsigned getPointerAddressSpace(Value *I);
- Align getAlign(LoadInst *LI) const {
- return DL.getValueOrABITypeAlignment(LI->getAlign(), LI->getType());
- }
-
- Align getAlign(StoreInst *SI) const {
- return DL.getValueOrABITypeAlignment(SI->getAlign(),
- SI->getValueOperand()->getType());
- }
-
static const unsigned MaxDepth = 3;
bool isConsecutiveAccess(Value *A, Value *B);
unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS);
unsigned VF = VecRegSize / Sz;
unsigned ChainSize = Chain.size();
- Align Alignment = getAlign(S0);
+ Align Alignment = S0->getAlign();
if (!isPowerOf2_32(Sz) || VF < 2 || ChainSize < 2) {
InstructionsProcessed->insert(Chain.begin(), Chain.end());
unsigned VecRegSize = TTI.getLoadStoreVecRegBitWidth(AS);
unsigned VF = VecRegSize / Sz;
unsigned ChainSize = Chain.size();
- Align Alignment = getAlign(L0);
+ Align Alignment = L0->getAlign();
if (!isPowerOf2_32(Sz) || VF < 2 || ChainSize < 2) {
InstructionsProcessed->insert(Chain.begin(), Chain.end());
// Arbitrarily try a vector of 2 elements.
Type *VecTy = VectorType::get(T, /*NumElements=*/2);
assert(VecTy && "did not find vectorized version of stored type");
- const MaybeAlign Alignment = getLoadStoreAlignment(ST);
- assert(Alignment && "Alignment should be set");
- if (!TTI->isLegalNTStore(VecTy, *Alignment)) {
+ if (!TTI->isLegalNTStore(VecTy, ST->getAlign())) {
reportVectorizationFailure(
"nontemporal store instruction cannot be vectorized",
"nontemporal store instruction cannot be vectorized",
// supported on the target (arbitrarily try a vector of 2 elements).
Type *VecTy = VectorType::get(I.getType(), /*NumElements=*/2);
assert(VecTy && "did not find vectorized version of load type");
- const MaybeAlign Alignment = getLoadStoreAlignment(LD);
- assert(Alignment && "Alignment should be set");
- if (!TTI->isLegalNTLoad(VecTy, *Alignment)) {
+ if (!TTI->isLegalNTLoad(VecTy, LD->getAlign())) {
reportVectorizationFailure(
"nontemporal load instruction cannot be vectorized",
"nontemporal load instruction cannot be vectorized",
if (!LI && !SI)
return false;
auto *Ty = getMemInstValueType(V);
- MaybeAlign Align = getLoadStoreAlignment(V);
+ Align Align = getLoadStoreAlignment(V);
return (LI && isLegalMaskedGather(Ty, Align)) ||
(SI && isLegalMaskedScatter(Ty, Align));
}
Type *ScalarDataTy = getMemInstValueType(Instr);
Type *DataTy = VectorType::get(ScalarDataTy, VF);
- // An alignment of 0 means target abi alignment. We need to use the scalar's
- // target abi alignment in such a case.
- const DataLayout &DL = Instr->getModule()->getDataLayout();
- const Align Alignment =
- DL.getValueOrABITypeAlignment(getLoadStoreAlignment(Instr), ScalarDataTy);
+ const Align Alignment = getLoadStoreAlignment(Instr);
// Determine if the pointer operand of the access is either consecutive or
// reverse consecutive.
"Widening decision should be ready at this moment");
return WideningDecision == CM_Scalarize;
}
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ? !(isLegalMaskedLoad(Ty, Ptr, Alignment) ||
isLegalMaskedGather(Ty, Alignment))
: !(isLegalMaskedStore(Ty, Ptr, Alignment) ||
"Masked interleave-groups for predicated accesses are not enabled.");
auto *Ty = getMemInstValueType(I);
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
return isa<LoadInst>(I) ? TTI.isLegalMaskedLoad(Ty, Alignment)
: TTI.isLegalMaskedStore(Ty, Alignment);
}
// Don't pass *I here, since it is scalar but will actually be part of a
// vectorized loop where the user of it is a vectorized instruction.
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
Cost += VF * TTI.getMemoryOpCost(I->getOpcode(), ValTy->getScalarType(),
Alignment, AS,
TTI::TCK_RecipThroughput);
assert((ConsecutiveStride == 1 || ConsecutiveStride == -1) &&
"Stride should be 1 or -1 for consecutive memory access");
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
unsigned Cost = 0;
if (Legal->isMaskRequired(I))
Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy,
- Alignment ? Alignment->value() : 0, AS,
- CostKind);
+ Alignment.value(), AS, CostKind);
else
Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS,
CostKind, I);
unsigned VF) {
Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
unsigned AS = getLoadStoreAddressSpace(I);
enum TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
if (isa<LoadInst>(I)) {
unsigned VF) {
Type *ValTy = getMemInstValueType(I);
auto *VectorTy = cast<VectorType>(ToVectorTy(ValTy, VF));
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
Value *Ptr = getLoadStorePointerOperand(I);
return TTI.getAddressComputationCost(VectorTy) +
TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,
- Legal->isMaskRequired(I),
- Alignment ? Alignment->value() : 0,
+ Legal->isMaskRequired(I), Alignment.value(),
TargetTransformInfo::TCK_RecipThroughput,
I);
}
// moment.
if (VF == 1) {
Type *ValTy = getMemInstValueType(I);
- const MaybeAlign Alignment = getLoadStoreAlignment(I);
+ const Align Alignment = getLoadStoreAlignment(I);
unsigned AS = getLoadStoreAddressSpace(I);
return TTI.getAddressComputationCost(ValTy) +
setInsertPointAfterBundle(E);
LoadInst *LI = cast<LoadInst>(VL0);
- Type *ScalarLoadTy = LI->getType();
unsigned AS = LI->getPointerAddressSpace();
Value *VecPtr = Builder.CreateBitCast(LI->getPointerOperand(),
if (getTreeEntry(PO))
ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
- Align Alignment = DL->getValueOrABITypeAlignment(LI->getAlign(),
- ScalarLoadTy);
- LI = Builder.CreateAlignedLoad(VecTy, VecPtr, Alignment);
+ LI = Builder.CreateAlignedLoad(VecTy, VecPtr, LI->getAlign());
Value *V = propagateMetadata(LI, E->Scalars);
if (IsReorder) {
SmallVector<int, 4> Mask;
bool IsReorder = !E->ReorderIndices.empty();
auto *SI = cast<StoreInst>(
IsReorder ? E->Scalars[E->ReorderIndices.front()] : VL0);
- unsigned Alignment = SI->getAlignment();
unsigned AS = SI->getPointerAddressSpace();
setInsertPointAfterBundle(E);
Value *ScalarPtr = SI->getPointerOperand();
Value *VecPtr = Builder.CreateBitCast(
ScalarPtr, VecValue->getType()->getPointerTo(AS));
- StoreInst *ST = Builder.CreateStore(VecValue, VecPtr);
+ StoreInst *ST = Builder.CreateAlignedStore(VecValue, VecPtr,
+ SI->getAlign());
// The pointer operand uses an in-tree scalar, so add the new BitCast to
// ExternalUses to make sure that an extract will be generated in the
if (getTreeEntry(ScalarPtr))
ExternalUses.push_back(ExternalUser(ScalarPtr, cast<User>(VecPtr), 0));
- if (!Alignment)
- Alignment = DL->getABITypeAlignment(SI->getValueOperand()->getType());
-
- ST->setAlignment(Align(Alignment));
Value *V = propagateMetadata(ST, E->Scalars);
if (NeedToShuffleReuses) {
V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),