void addMemSet(int64_t OffsetFromFirst, MemSetInst *MSI) {
int64_t Size = cast<ConstantInt>(MSI->getLength())->getZExtValue();
- addRange(OffsetFromFirst, Size, MSI->getDest(), MSI->getAlignment(), MSI);
+ addRange(OffsetFromFirst, Size, MSI->getDest(), MSI->getDestAlignment(), MSI);
}
void addRange(int64_t Start, int64_t Size, Value *Ptr,
return AMemSet;
}
-static unsigned findCommonAlignment(const DataLayout &DL, const StoreInst *SI,
- const LoadInst *LI) {
+static unsigned findStoreAlignment(const DataLayout &DL, const StoreInst *SI) {
unsigned StoreAlign = SI->getAlignment();
if (!StoreAlign)
StoreAlign = DL.getABITypeAlignment(SI->getOperand(0)->getType());
+ return StoreAlign;
+}
+
+static unsigned findLoadAlignment(const DataLayout &DL, const LoadInst *LI) {
unsigned LoadAlign = LI->getAlignment();
if (!LoadAlign)
LoadAlign = DL.getABITypeAlignment(LI->getType());
+ return LoadAlign;
+}
- return std::min(StoreAlign, LoadAlign);
+static unsigned findCommonAlignment(const DataLayout &DL, const StoreInst *SI,
+ const LoadInst *LI) {
+ unsigned StoreAlign = findStoreAlignment(DL, SI);
+ unsigned LoadAlign = findLoadAlignment(DL, LI);
+ return MinAlign(StoreAlign, LoadAlign);
}
// This method try to lift a store instruction before position P.
if (!AA.isNoAlias(MemoryLocation::get(SI), LoadLoc))
UseMemMove = true;
- unsigned Align = findCommonAlignment(DL, SI, LI);
uint64_t Size = DL.getTypeStoreSize(T);
IRBuilder<> Builder(P);
Instruction *M;
if (UseMemMove)
- M = Builder.CreateMemMove(SI->getPointerOperand(),
- LI->getPointerOperand(), Size,
- Align, SI->isVolatile());
+ M = Builder.CreateMemMove(
+ SI->getPointerOperand(), findStoreAlignment(DL, SI),
+ LI->getPointerOperand(), findLoadAlignment(DL, LI), Size,
+ SI->isVolatile());
else
- M = Builder.CreateMemCpy(SI->getPointerOperand(),
- LI->getPointerOperand(), Size,
- Align, SI->isVolatile());
+ M = Builder.CreateMemCpy(
+ SI->getPointerOperand(), findStoreAlignment(DL, SI),
+ LI->getPointerOperand(), findLoadAlignment(DL, LI), Size,
+ SI->isVolatile());
DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI
<< " => " << *M << "\n");
// If all checks passed, then we can transform M.
- // Make sure to use the lesser of the alignment of the source and the dest
- // since we're changing where we're reading from, but don't want to increase
- // the alignment past what can be read from or written to.
// TODO: Is this worth it if we're creating a less aligned memcpy? For
// example we could be moving from movaps -> movq on x86.
- unsigned Align = std::min(MDep->getAlignment(), M->getAlignment());
-
IRBuilder<> Builder(M);
if (UseMemMove)
- Builder.CreateMemMove(M->getRawDest(), MDep->getRawSource(), M->getLength(),
- Align, M->isVolatile());
+ Builder.CreateMemMove(M->getRawDest(), M->getDestAlignment(),
+ MDep->getRawSource(), MDep->getSourceAlignment(),
+ M->getLength(), M->isVolatile());
else
- Builder.CreateMemCpy(M->getRawDest(), MDep->getRawSource(), M->getLength(),
- Align, M->isVolatile());
+ Builder.CreateMemCpy(M->getRawDest(), M->getDestAlignment(),
+ MDep->getRawSource(), MDep->getSourceAlignment(),
+ M->getLength(), M->isVolatile());
// Remove the instruction we're replacing.
MD->removeInstruction(M);
// If Dest is aligned, and SrcSize is constant, use the minimum alignment
// of the sum.
const unsigned DestAlign =
- std::max(MemSet->getAlignment(), MemCpy->getAlignment());
+ std::max(MemSet->getDestAlignment(), MemCpy->getDestAlignment());
if (DestAlign > 1)
if (ConstantInt *SrcSizeC = dyn_cast<ConstantInt>(SrcSize))
Align = MinAlign(SrcSizeC->getZExtValue(), DestAlign);
IRBuilder<> Builder(MemCpy);
Builder.CreateMemSet(MemCpy->getRawDest(), MemSet->getOperand(1),
- CopySize, MemCpy->getAlignment());
+ CopySize, MemCpy->getDestAlignment());
return true;
}
if (Value *ByteVal = isBytewiseValue(GV->getInitializer())) {
IRBuilder<> Builder(M);
Builder.CreateMemSet(M->getRawDest(), ByteVal, M->getLength(),
- M->getAlignment(), false);
+ M->getDestAlignment(), false);
MD->removeInstruction(M);
M->eraseFromParent();
++NumCpyToSet;
// d) memcpy from a just-memset'd source can be turned into memset.
if (DepInfo.isClobber()) {
if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
+ // FIXME: Can we pass in either of dest/src alignment here instead
+ // of conservatively taking the minimum?
+ unsigned Align = MinAlign(M->getDestAlignment(), M->getSourceAlignment());
if (performCallSlotOptzn(M, M->getDest(), M->getSource(),
- CopySize->getZExtValue(), M->getAlignment(),
+ CopySize->getZExtValue(), Align,
C)) {
MD->removeInstruction(M);
M->eraseFromParent();
// source of the memcpy to the alignment we need. If we fail, we bail out.
AssumptionCache &AC = LookupAssumptionCache();
DominatorTree &DT = LookupDomTree();
- if (MDep->getAlignment() < ByValAlign &&
+ if (MDep->getSourceAlignment() < ByValAlign &&
getOrEnforceKnownAlignment(MDep->getSource(), ByValAlign, DL,
CS.getInstruction(), &AC, &DT) < ByValAlign)
return false;
%T = type { i8, i32 }
+; Ensure load-store forwarding of an aggregate is interpreted as
+; a memmove when the source and dest may alias
+define void @test_memmove(%T* align 8 %a, %T* align 16 %b) {
+; CHECK-LABEL: @test_memmove(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast %T* [[B:%.*]] to i8*
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast %T* [[A:%.*]] to i8*
+; CHECK-NEXT: call void @llvm.memmove.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
+; CHECK-NEXT: ret void
+;
+ %val = load %T, %T* %a, align 8
+ store %T %val, %T* %b, align 16
+ ret void
+}
+
+; Ensure load-store forwarding of an aggregate is interpreted as
+; a memcpy when the source and dest do not alias
+define void @test_memcpy(%T* noalias align 8 %a, %T* noalias align 16 %b) {
+; CHECK-LABEL: @test_memcpy(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast %T* [[B:%.*]] to i8*
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast %T* [[A:%.*]] to i8*
+; CHECK-NEXT: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 [[TMP1]], i8* align 8 [[TMP2]], i64 8, i1 false)
+; CHECK-NEXT: ret void
+;
+ %val = load %T, %T* %a, align 8
+ store %T %val, %T* %b, align 16
+ ret void
+}
+
; memcpy(%d, %a) should not be generated since store2 may-aliases load %a.
define void @f(%T* %a, %T* %b, %T* %c, %T* %d) {
; CHECK-LABEL: @f(