#include "llvm/IR/Instructions.h"
#include "llvm/IR/PassManager.h"
#include "llvm/Pass.h"
+#include "llvm/Support/Allocator.h"
#include <algorithm>
+#include <utility>
namespace llvm {
SmallPtrSet<const BlockT *, 8> DenseBlockSet;
+#if !defined(NDEBUG) || !LLVM_ENABLE_ABI_BREAKING_CHECKS
/// Indicator that this loop is no longer a valid loop.
bool IsInvalid = false;
+#endif
LoopBase(const LoopBase<BlockT, LoopT> &) = delete;
const LoopBase<BlockT, LoopT> &
operator=(const LoopBase<BlockT, LoopT> &) = delete;
- void clear() {
- IsInvalid = true;
- SubLoops.clear();
- Blocks.clear();
- DenseBlockSet.clear();
- ParentLoop = nullptr;
- }
-
public:
- /// This creates an empty loop.
- LoopBase() : ParentLoop(nullptr) {}
-
/// Return the nesting level of this loop. An outer-most loop has depth 1,
/// for consistency with loop depth values used for basic blocks, where depth
/// 0 is used for blocks not inside any loops.
return Blocks.size();
}
- /// Return true if this loop is no longer valid.
+#ifndef NDEBUG
+ /// Return true if this loop is no longer valid. The only valid use of this
+ /// helper is "assert(L.isInvalid())" or equivalent, since IsInvalid is set to
+ /// false by the destructor. In other words, if this accessor returns false,
+ /// the caller has already triggered UB by calling this accessor; and so it
+ /// can only be called in a context where a return value of false indicates a
+ /// programmer error.
bool isInvalid() const { return IsInvalid; }
+#endif
/// True if terminator in the block can branch to another block that is
/// outside of the current loop.
protected:
friend class LoopInfoBase<BlockT, LoopT>;
+
+ /// This creates an empty loop.
+ LoopBase() : ParentLoop(nullptr) {}
+
explicit LoopBase(BlockT *BB) : ParentLoop(nullptr) {
Blocks.push_back(BB);
DenseBlockSet.insert(BB);
// non-public.
~LoopBase() {
for (auto *SubLoop : SubLoops)
- delete SubLoop;
+ SubLoop->~LoopT();
+
+ IsInvalid = true;
+ SubLoops.clear();
+ Blocks.clear();
+ DenseBlockSet.clear();
+ ParentLoop = nullptr;
}
};
explicit operator bool() const { return Start && End; }
};
- Loop() {}
-
/// Return true if the specified value is loop invariant.
bool isLoopInvariant(const Value *V) const;
LocRange getLocRange() const;
StringRef getName() const {
- if (isInvalid())
- return "<invalidated loop>";
if (BasicBlock *Header = getHeader())
if (Header->hasName())
return Header->getName();
}
private:
+ Loop() = default;
+
friend class LoopInfoBase<BasicBlock, Loop>;
friend class LoopBase<BasicBlock, Loop>;
explicit Loop(BasicBlock *BB) : LoopBase<BasicBlock, Loop>(BB) {}
// BBMap - Mapping of basic blocks to the inner most loop they occur in
DenseMap<const BlockT *, LoopT *> BBMap;
std::vector<LoopT *> TopLevelLoops;
- std::vector<LoopT *> RemovedLoops;
+ BumpPtrAllocator LoopAllocator;
friend class LoopBase<BlockT, LoopT>;
friend class LoopInfo;
LoopInfoBase(LoopInfoBase &&Arg)
: BBMap(std::move(Arg.BBMap)),
- TopLevelLoops(std::move(Arg.TopLevelLoops)) {
+ TopLevelLoops(std::move(Arg.TopLevelLoops)),
+ LoopAllocator(std::move(Arg.LoopAllocator)) {
// We have to clear the arguments top level loops as we've taken ownership.
Arg.TopLevelLoops.clear();
}
BBMap = std::move(RHS.BBMap);
for (auto *L : TopLevelLoops)
- delete L;
+ L->~LoopT();
+
TopLevelLoops = std::move(RHS.TopLevelLoops);
+ LoopAllocator = std::move(RHS.LoopAllocator);
RHS.TopLevelLoops.clear();
return *this;
}
BBMap.clear();
for (auto *L : TopLevelLoops)
- delete L;
+ L->~LoopT();
TopLevelLoops.clear();
- for (auto *L : RemovedLoops)
- delete L;
- RemovedLoops.clear();
+ LoopAllocator.Reset();
+ }
+
+ template <typename... ArgsTy> LoopT *AllocateLoop(ArgsTy &&... Args) {
+ LoopT *Storage = LoopAllocator.Allocate<LoopT>();
+ return new (Storage) LoopT(std::forward<ArgsTy>(Args)...);
}
/// iterator/begin/end - The interface to the top-level loops in the current
void verify(const DominatorTreeBase<BlockT, false> &DomTree) const;
protected:
- static void clearLoop(LoopT &L) { L.clear(); }
+ // Calls the destructor for \p L but keeps the memory for \p L around so that
+ // the pointer value does not get re-used.
+ void destroy(LoopT *L) {
+ L->~LoopT();
+
+ // Since LoopAllocator is a BumpPtrAllocator, this Deallocate only poisons
+ // \c L, but the pointer remains valid for non-dereferencing uses.
+ LoopAllocator.Deallocate(L, sizeof(LoopT));
+ }
};
// Implementation in LoopInfoImpl.h
}
// Perform a backward CFG traversal to discover and map blocks in this loop.
if (!Backedges.empty()) {
- LoopT *L = new LoopT(Header);
+ LoopT *L = AllocateLoop(Header);
discoverAndMapSubloop(L, ArrayRef<BlockT *>(Backedges), this, DomTree);
}
}
// Add a new loop into the loop queue.
void addLoop(Loop &L);
+ // Mark \p L as deleted.
+ void markLoopAsDeleted(Loop &L);
+
//===--------------------------------------------------------------------===//
/// SimpleAnalysis - Provides simple interface to update analysis info
/// maintained by various passes. Note, if required this interface can
std::deque<Loop *> LQ;
LoopInfo *LI;
Loop *CurrentLoop;
+ bool CurrentLoopDeleted;
};
// This pass is required by the LCSSA transformation. It is used inside
class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
public:
- MachineLoop();
-
/// Return the "top" block in the loop, which is the first block in the linear
/// layout, ignoring any parts of the loop not contiguous with the part that
/// contains the header.
explicit MachineLoop(MachineBasicBlock *MBB)
: LoopBase<MachineBasicBlock, MachineLoop>(MBB) {}
+
+ MachineLoop() = default;
};
// Implementation in LoopInfoImpl.h
/// This doesn't invalidate, but instead simply deletes, the relevant results.
/// It is useful when the IR is being removed and we want to clear out all the
/// memory pinned for it.
- void clear(IRUnitT &IR) {
+ void clear(IRUnitT &IR, llvm::StringRef Name) {
if (DebugLogging)
- dbgs() << "Clearing all analysis results for: " << IR.getName() << "\n";
+ dbgs() << "Clearing all analysis results for: " << Name << "\n";
auto ResultsListI = AnalysisResultLists.find(&IR);
if (ResultsListI == AnalysisResultLists.end())
// Pull in base class overloads.
using AllocatorBase<BumpPtrAllocatorImpl>::Allocate;
+ // Bump pointer allocators are expected to never free their storage; and
+ // clients expect pointers to remain valid for non-dereferencing uses even
+ // after deallocation.
void Deallocate(const void *Ptr, size_t Size) {
__asan_poison_memory_region(Ptr, Size);
}
/// If this is called for the current loop, in addition to clearing any
/// state, this routine will mark that the current loop should be skipped by
/// the rest of the pass management infrastructure.
- void markLoopAsDeleted(Loop &L) {
- LAM.clear(L);
+ void markLoopAsDeleted(Loop &L, llvm::StringRef Name) {
+ LAM.clear(L, Name);
assert((&L == CurrentL || CurrentL->contains(&L)) &&
"Cannot delete a loop outside of the "
"subloop tree currently being processed.");
auto PAC = PA.getChecker<FunctionAnalysisManagerCGSCCProxy>();
if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<LazyCallGraph::SCC>>()) {
for (LazyCallGraph::Node &N : C)
- FAM->clear(N.getFunction());
+ FAM->clear(N.getFunction(), N.getFunction().getName());
return true;
}
// those results. Note that the order doesn't matter here as this will just
// directly destroy the results without calling methods on them.
for (Loop *L : PreOrderLoops)
- InnerAM->clear(*L);
+ InnerAM->clear(*L, L->getName());
// We also need to null out the inner AM so that when the object gets
// destroyed as invalid we don't try to clear the inner AM again. At that
void LoopInfo::erase(Loop *Unloop) {
assert(!Unloop->isInvalid() && "Loop has already been erased!");
- RemovedLoops.push_back(Unloop);
- auto InvalidateOnExit =
- make_scope_exit([&]() { BaseT::clearLoop(*Unloop); });
+ auto InvalidateOnExit = make_scope_exit([&]() { destroy(Unloop); });
// First handle the special case of no parent loop to simplify the algorithm.
if (!Unloop->getParentLoop()) {
Info.setPreservesAll();
}
+void LPPassManager::markLoopAsDeleted(Loop &L) {
+ assert((&L == CurrentLoop || CurrentLoop->contains(&L)) &&
+ "Must not delete loop outside the current loop tree!");
+ if (&L == CurrentLoop)
+ CurrentLoopDeleted = true;
+}
+
/// run - Execute all of the passes scheduled for execution. Keep track of
/// whether any of the passes modifies the function, and if so, return true.
bool LPPassManager::runOnFunction(Function &F) {
// Walk Loops
while (!LQ.empty()) {
- bool LoopWasDeleted = false;
+ CurrentLoopDeleted = false;
CurrentLoop = LQ.back();
// Run all passes on the current Loop.
Changed |= P->runOnLoop(CurrentLoop, *this);
}
- LoopWasDeleted = CurrentLoop->isInvalid();
if (Changed)
- dumpPassInfo(P, MODIFICATION_MSG, ON_LOOP_MSG, CurrentLoop->getName());
+ dumpPassInfo(P, MODIFICATION_MSG, ON_LOOP_MSG,
+ CurrentLoopDeleted ? "<deleted loop>"
+ : CurrentLoop->getName());
dumpPreservedSet(P);
- if (LoopWasDeleted) {
+ if (CurrentLoopDeleted) {
// Notify passes that the loop is being deleted.
deleteSimpleAnalysisLoop(CurrentLoop);
} else {
removeNotPreservedAnalysis(P);
recordAvailableAnalysis(P);
- removeDeadPasses(P, LoopWasDeleted ? "<deleted>"
- : CurrentLoop->getHeader()->getName(),
+ removeDeadPasses(P,
+ CurrentLoopDeleted ? "<deleted>"
+ : CurrentLoop->getHeader()->getName(),
ON_LOOP_MSG);
- if (LoopWasDeleted)
+ if (CurrentLoopDeleted)
// Do not run other passes on this loop.
break;
}
// If the loop was deleted, release all the loop passes. This frees up
// some memory, and avoids trouble with the pass manager trying to call
// verifyAnalysis on them.
- if (LoopWasDeleted) {
+ if (CurrentLoopDeleted) {
for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) {
Pass *P = getContainedPass(Index);
freePass(P, "<deleted>", ON_LOOP_MSG);
char LCSSAVerificationPass::ID = 0;
INITIALIZE_PASS(LCSSAVerificationPass, "lcssa-verification", "LCSSA Verifier",
false, false)
-
FunctionAnalysisManager &FAM =
AM.getResult<FunctionAnalysisManagerCGSCCProxy>(DeadC, CG)
.getManager();
- FAM.clear(*DeadF);
- AM.clear(DeadC);
+ FAM.clear(*DeadF, DeadF->getName());
+ AM.clear(DeadC, DeadC.getName());
auto &DeadRC = DeadC.getOuterRefSCC();
CG.removeDeadFunction(*DeadF);
//
Pass *llvm::createLoopExtractorPass() { return new LoopExtractor(); }
-bool LoopExtractor::runOnLoop(Loop *L, LPPassManager &) {
+bool LoopExtractor::runOnLoop(Loop *L, LPPassManager &LPM) {
if (skipLoop(L))
return false;
Changed = true;
// After extraction, the loop is replaced by a function call, so
// we shouldn't try to run any more loop passes on it.
+ LPM.markLoopAsDeleted(*L);
LI.erase(L);
}
++NumExtracted;
Loop *LoopConstrainer::createClonedLoopStructure(Loop *Original, Loop *Parent,
ValueToValueMapTy &VM) {
- Loop &New = *new Loop();
+ Loop &New = *LI.AllocateLoop();
if (Parent)
Parent->addChildLoop(&New);
else
DEBUG(dbgs() << "Analyzing Loop for deletion: ");
DEBUG(L.dump());
+ std::string LoopName = L.getName();
auto Result = deleteLoopIfDead(&L, AR.DT, AR.SE, AR.LI);
if (Result == LoopDeletionResult::Unmodified)
return PreservedAnalyses::all();
if (Result == LoopDeletionResult::Deleted)
- Updater.markLoopAsDeleted(L);
+ Updater.markLoopAsDeleted(L, LoopName);
return getLoopPassPreservedAnalyses();
}
Pass *llvm::createLoopDeletionPass() { return new LoopDeletionLegacyPass(); }
-bool LoopDeletionLegacyPass::runOnLoop(Loop *L, LPPassManager &) {
+bool LoopDeletionLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) {
if (skipLoop(L))
return false;
DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
DEBUG(dbgs() << "Analyzing Loop for deletion: ");
DEBUG(L->dump());
- return deleteLoopIfDead(L, DT, SE, LI) != LoopDeletionResult::Unmodified;
+
+ LoopDeletionResult Result = deleteLoopIfDead(L, DT, SE, LI);
+
+ if (Result == LoopDeletionResult::Deleted)
+ LPM.markLoopAsDeleted(*L);
+
+ return Result != LoopDeletionResult::Unmodified;
}
Optional<bool> ProvidedUpperBound;
Optional<bool> ProvidedAllowPeeling;
- bool runOnLoop(Loop *L, LPPassManager &) override {
+ bool runOnLoop(Loop *L, LPPassManager &LPM) override {
if (skipLoop(L))
return false;
OptimizationRemarkEmitter ORE(&F);
bool PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
- return tryToUnrollLoop(L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel,
- ProvidedCount, ProvidedThreshold,
- ProvidedAllowPartial, ProvidedRuntime,
- ProvidedUpperBound, ProvidedAllowPeeling) !=
- LoopUnrollResult::Unmodified;
+ LoopUnrollResult Result = tryToUnrollLoop(
+ L, DT, LI, SE, TTI, AC, ORE, PreserveLCSSA, OptLevel, ProvidedCount,
+ ProvidedThreshold, ProvidedAllowPartial, ProvidedRuntime,
+ ProvidedUpperBound, ProvidedAllowPeeling);
+
+ if (Result == LoopUnrollResult::FullyUnrolled)
+ LPM.markLoopAsDeleted(*L);
+
+ return Result != LoopUnrollResult::Unmodified;
}
/// This transformation requires natural loop information & requires that
else
OldLoops.insert(AR.LI.begin(), AR.LI.end());
+ std::string LoopName = L.getName();
+
bool Changed =
tryToUnrollLoop(&L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, *ORE,
/*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
Updater.addSiblingLoops(SibLoops);
if (!IsCurrentLoopValid) {
- Updater.markLoopAsDeleted(L);
+ Updater.markLoopAsDeleted(L, LoopName);
} else {
// We can only walk child loops if the current loop remained valid.
if (UnrollRevisitChildLoops) {
// bloating it further.
if (PSI && PSI->hasHugeWorkingSetSize())
AllowPeeling = false;
+ std::string LoopName = L.getName();
LoopUnrollResult Result =
tryToUnrollLoop(&L, DT, &LI, SE, TTI, AC, ORE,
/*PreserveLCSSA*/ true, OptLevel, /*Count*/ None,
// Clear any cached analysis results for L if we removed it completely.
if (LAM && Result == LoopUnrollResult::FullyUnrolled)
- LAM->clear(L);
+ LAM->clear(L, LoopName);
}
if (!Changed)
/// mapping the blocks with the specified map.
static Loop *CloneLoop(Loop *L, Loop *PL, ValueToValueMapTy &VM,
LoopInfo *LI, LPPassManager *LPM) {
- Loop &New = *new Loop();
+ Loop &New = *LI->AllocateLoop();
if (PL)
PL->addChildLoop(&New);
else
Function *F = OrigLoop->getHeader()->getParent();
Loop *ParentLoop = OrigLoop->getParentLoop();
- Loop *NewLoop = new Loop();
+ Loop *NewLoop = LI->AllocateLoop();
if (ParentLoop)
ParentLoop->addChildLoop(NewLoop);
else
placeSplitBlockCarefully(NewBB, OuterLoopPreds, L);
// Create the new outer loop.
- Loop *NewOuter = new Loop();
+ Loop *NewOuter = LI->AllocateLoop();
// Change the parent loop to use the outer loop as its child now.
if (Loop *Parent = L->getParentLoop())
assert(OriginalBB == OldLoop->getHeader() &&
"Header should be first in RPO");
- NewLoop = new Loop();
+ NewLoop = LI->AllocateLoop();
Loop *NewLoopParent = NewLoops.lookup(OldLoop->getParentLoop());
if (NewLoopParent)
MiddleBlock->splitBasicBlock(MiddleBlock->getTerminator(), "scalar.ph");
// Create and register the new vector loop.
- Loop *Lp = new Loop();
+ Loop *Lp = LI->AllocateLoop();
Loop *ParentLoop = OrigLoop->getParentLoop();
// Insert the new loop into the loop nest and register the new basic blocks
; CHECK: Running analysis: LoopAccessAnalysis on outer.header
; CHECK: Finished Loop pass manager run.
; CHECK: Running pass: LoopUnrollPass
-; CHECK: Clearing all analysis results for: <invalidated loop>
-; CHECK: Clearing all analysis results for: <invalidated loop>
+; CHECK: Clearing all analysis results for: inner2.header
+; CHECK: Clearing all analysis results for: outer.header
; CHECK: Invalidating all non-preserved analyses for: test
; CHECK: Invalidating all non-preserved analyses for: inner1.header
; CHECK: Invalidating analysis: LoopAccessAnalysis on inner1.header
.WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &Updater) {
- auto *NewLoop = new Loop();
+ auto *NewLoop = AR.LI.AllocateLoop();
L.addChildLoop(NewLoop);
auto *NewLoop010PHBB =
BasicBlock::Create(Context, "loop.0.1.0.ph", &F, &Loop02PHBB);
.WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &Updater) {
- auto *NewLoop = new Loop();
+ auto *NewLoop = AR.LI.AllocateLoop();
L.addChildLoop(NewLoop);
auto *NewLoop011PHBB = BasicBlock::Create(Context, "loop.0.1.1.ph", &F, NewLoop01LatchBB);
auto *NewLoop011BB = BasicBlock::Create(Context, "loop.0.1.1", &F, NewLoop01LatchBB);
.WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &Updater) {
- auto *NewLoop = new Loop();
+ auto *NewLoop = AR.LI.AllocateLoop();
L.getParentLoop()->addChildLoop(NewLoop);
auto *NewLoop01PHBB = BasicBlock::Create(Context, "loop.0.1.ph", &F, &Loop02PHBB);
auto *NewLoop01BB = BasicBlock::Create(Context, "loop.0.1", &F, &Loop02PHBB);
.WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &Updater) {
- Loop *NewLoops[] = {new Loop(), new Loop(), new Loop()};
+ Loop *NewLoops[] = {AR.LI.AllocateLoop(), AR.LI.AllocateLoop(),
+ AR.LI.AllocateLoop()};
L.getParentLoop()->addChildLoop(NewLoops[0]);
L.getParentLoop()->addChildLoop(NewLoops[1]);
NewLoops[1]->addChildLoop(NewLoops[2]);
.WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
LoopStandardAnalysisResults &AR,
LPMUpdater &Updater) {
- auto *NewLoop = new Loop();
+ auto *NewLoop = AR.LI.AllocateLoop();
AR.LI.addTopLevelLoop(NewLoop);
auto *NewLoop1PHBB = BasicBlock::Create(Context, "loop.1.ph", &F, &Loop2BB);
auto *NewLoop1BB = BasicBlock::Create(Context, "loop.1", &F, &Loop2BB);
LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
assert(L.empty() && "Can only delete leaf loops with this routine!");
SmallVector<BasicBlock *, 4> LoopBBs(L.block_begin(), L.block_end());
- Updater.markLoopAsDeleted(L);
+ Updater.markLoopAsDeleted(L, L.getName());
IDomBB.getTerminator()->replaceUsesOfWith(L.getHeader(),
L.getUniqueExitBlock());
for (BasicBlock *LoopBB : LoopBBs) {
EraseLoop(L, Loop02PHBB, AR, Updater);
// Now insert a new sibling loop.
- auto *NewSibling = new Loop;
+ auto *NewSibling = AR.LI.AllocateLoop();
ParentL->addChildLoop(NewSibling);
NewLoop03PHBB =
BasicBlock::Create(Context, "loop.0.3.ph", &F, &Loop0LatchBB);