lib/Passes/PassBuilder.cpp

   1 //===- Parsing, selection, and construction of pass pipelines -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 ///
  10 /// This file provides the implementation of the PassBuilder based on our
  11 /// static pass registry as well as related functionality. It also provides
  12 /// helpers to aid in analyzing, debugging, and testing passes and pass
  13 /// pipelines.
  14 ///
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "llvm/Passes/PassBuilder.h"
  18 #include "llvm/ADT/StringSwitch.h"
  19 #include "llvm/Analysis/AliasAnalysis.h"
  20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
  21 #include "llvm/Analysis/AssumptionCache.h"
  22 #include "llvm/Analysis/BasicAliasAnalysis.h"
  23 #include "llvm/Analysis/BlockFrequencyInfo.h"
  24 #include "llvm/Analysis/BranchProbabilityInfo.h"
  25 #include "llvm/Analysis/CFGPrinter.h"
  26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
  27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
  28 #include "llvm/Analysis/CGSCCPassManager.h"
  29 #include "llvm/Analysis/CallGraph.h"
  30 #include "llvm/Analysis/DemandedBits.h"
  31 #include "llvm/Analysis/DependenceAnalysis.h"
  32 #include "llvm/Analysis/DominanceFrontier.h"
  33 #include "llvm/Analysis/GlobalsModRef.h"
  34 #include "llvm/Analysis/IVUsers.h"
  35 #include "llvm/Analysis/LazyCallGraph.h"
  36 #include "llvm/Analysis/LazyValueInfo.h"
  37 #include "llvm/Analysis/LoopAccessAnalysis.h"
  38 #include "llvm/Analysis/LoopInfo.h"
  39 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
  40 #include "llvm/Analysis/MemorySSA.h"
  41 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  42 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  43 #include "llvm/Analysis/PhiValues.h"
  44 #include "llvm/Analysis/PostDominators.h"
  45 #include "llvm/Analysis/ProfileSummaryInfo.h"
  46 #include "llvm/Analysis/RegionInfo.h"
  47 #include "llvm/Analysis/ScalarEvolution.h"
  48 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
  49 #include "llvm/Analysis/ScopedNoAliasAA.h"
  50 #include "llvm/Analysis/StackSafetyAnalysis.h"
  51 #include "llvm/Analysis/TargetLibraryInfo.h"
  52 #include "llvm/Analysis/TargetTransformInfo.h"
  53 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
  54 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
  55 #include "llvm/CodeGen/UnreachableBlockElim.h"
  56 #include "llvm/IR/Dominators.h"
  57 #include "llvm/IR/IRPrintingPasses.h"
  58 #include "llvm/IR/PassManager.h"
  59 #include "llvm/IR/SafepointIRVerifier.h"
  60 #include "llvm/IR/Verifier.h"
  61 #include "llvm/Support/Debug.h"
  62 #include "llvm/Support/FormatVariadic.h"
  63 #include "llvm/Support/Regex.h"
  64 #include "llvm/Target/TargetMachine.h"
  65 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
  66 #include "llvm/Transforms/IPO/AlwaysInliner.h"
  67 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
  68 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
  69 #include "llvm/Transforms/IPO/ConstantMerge.h"
  70 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
  71 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
  72 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
  73 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
  74 #include "llvm/Transforms/IPO/FunctionAttrs.h"
  75 #include "llvm/Transforms/IPO/FunctionImport.h"
  76 #include "llvm/Transforms/IPO/GlobalDCE.h"
  77 #include "llvm/Transforms/IPO/GlobalOpt.h"
  78 #include "llvm/Transforms/IPO/GlobalSplit.h"
  79 #include "llvm/Transforms/IPO/HotColdSplitting.h"
  80 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
  81 #include "llvm/Transforms/IPO/Inliner.h"
  82 #include "llvm/Transforms/IPO/Internalize.h"
  83 #include "llvm/Transforms/IPO/LowerTypeTests.h"
  84 #include "llvm/Transforms/IPO/PartialInlining.h"
  85 #include "llvm/Transforms/IPO/SCCP.h"
  86 #include "llvm/Transforms/IPO/SampleProfile.h"
  87 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
  88 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
  89 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  90 #include "llvm/Transforms/InstCombine/InstCombine.h"
  91 #include "llvm/Transforms/Instrumentation.h"
  92 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
  93 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
  94 #include "llvm/Transforms/Instrumentation/CGProfile.h"
  95 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
  96 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
  97 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
  98 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
  99 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 100 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 101 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 102 #include "llvm/Transforms/Scalar/ADCE.h"
 103 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
 104 #include "llvm/Transforms/Scalar/BDCE.h"
 105 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
 106 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
 107 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
 108 #include "llvm/Transforms/Scalar/DCE.h"
 109 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
 110 #include "llvm/Transforms/Scalar/DivRemPairs.h"
 111 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 112 #include "llvm/Transforms/Scalar/Float2Int.h"
 113 #include "llvm/Transforms/Scalar/GVN.h"
 114 #include "llvm/Transforms/Scalar/GuardWidening.h"
 115 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
 116 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
 117 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
 118 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
 119 #include "llvm/Transforms/Scalar/JumpThreading.h"
 120 #include "llvm/Transforms/Scalar/LICM.h"
 121 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
 122 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
 123 #include "llvm/Transforms/Scalar/LoopDeletion.h"
 124 #include "llvm/Transforms/Scalar/LoopDistribute.h"
 125 #include "llvm/Transforms/Scalar/LoopFuse.h"
 126 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
 127 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
 128 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
 129 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 130 #include "llvm/Transforms/Scalar/LoopPredication.h"
 131 #include "llvm/Transforms/Scalar/LoopRotation.h"
 132 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
 133 #include "llvm/Transforms/Scalar/LoopSink.h"
 134 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
 135 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
 136 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
 137 #include "llvm/Transforms/Scalar/LowerAtomic.h"
 138 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 139 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
 140 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
 141 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
 142 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
 143 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
 144 #include "llvm/Transforms/Scalar/NaryReassociate.h"
 145 #include "llvm/Transforms/Scalar/NewGVN.h"
 146 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
 147 #include "llvm/Transforms/Scalar/Reassociate.h"
 148 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
 149 #include "llvm/Transforms/Scalar/SCCP.h"
 150 #include "llvm/Transforms/Scalar/SROA.h"
 151 #include "llvm/Transforms/Scalar/Scalarizer.h"
 152 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
 153 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
 154 #include "llvm/Transforms/Scalar/Sink.h"
 155 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
 156 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
 157 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
 158 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
 159 #include "llvm/Transforms/Utils/AddDiscriminators.h"
 160 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
 161 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 162 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 163 #include "llvm/Transforms/Utils/LCSSA.h"
 164 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 165 #include "llvm/Transforms/Utils/LoopSimplify.h"
 166 #include "llvm/Transforms/Utils/LowerInvoke.h"
 167 #include "llvm/Transforms/Utils/Mem2Reg.h"
 168 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 169 #include "llvm/Transforms/Utils/SymbolRewriter.h"
 170 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 171 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 172 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
 173
 174 using namespace llvm;
 175
 176 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
 177                                              cl::ReallyHidden, cl::init(4));
 178 static cl::opt<bool>
 179     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
 180                        cl::Hidden, cl::ZeroOrMore,
 181                        cl::desc("Run Partial inlinining pass"));
 182
 183 static cl::opt<bool>
 184     RunNewGVN("enable-npm-newgvn", cl::init(false),
 185               cl::Hidden, cl::ZeroOrMore,
 186               cl::desc("Run NewGVN instead of GVN"));
 187
 188 static cl::opt<bool> EnableEarlyCSEMemSSA(
 189     "enable-npm-earlycse-memssa", cl::init(true), cl::Hidden,
 190     cl::desc("Enable the EarlyCSE w/ MemorySSA pass for the new PM (default = on)"));
 191
 192 static cl::opt<bool> EnableGVNHoist(
 193     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
 194     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 195
 196 static cl::opt<bool> EnableGVNSink(
 197     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
 198     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 199
 200 static cl::opt<bool> EnableUnrollAndJam(
 201     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
 202     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
 203
 204 static cl::opt<bool> EnableSyntheticCounts(
 205     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
 206     cl::desc("Run synthetic function entry count generation "
 207              "pass"));
 208
 209 static Regex DefaultAliasRegex(
 210     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
 211
 212 // This option is used in simplifying testing SampleFDO optimizations for
 213 // profile loading.
 214 static cl::opt<bool>
 215     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
 216               cl::desc("Enable control height reduction optimization (CHR)"));
 217
 218 extern cl::opt<bool> EnableHotColdSplit;
 219 extern cl::opt<bool> EnableOrderFileInstrumentation;
 220
 221 extern cl::opt<bool> FlattenedProfileUsed;
 222
 223 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
 224   switch (Level) {
 225   case PassBuilder::O0:
 226   case PassBuilder::O1:
 227   case PassBuilder::O2:
 228   case PassBuilder::O3:
 229     return false;
 230
 231   case PassBuilder::Os:
 232   case PassBuilder::Oz:
 233     return true;
 234   }
 235   llvm_unreachable("Invalid optimization level!");
 236 }
 237
 238 namespace {
 239
 240 /// No-op module pass which does nothing.
 241 struct NoOpModulePass {
 242   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
 243     return PreservedAnalyses::all();
 244   }
 245   static StringRef name() { return "NoOpModulePass"; }
 246 };
 247
 248 /// No-op module analysis.
 249 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
 250   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
 251   static AnalysisKey Key;
 252
 253 public:
 254   struct Result {};
 255   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
 256   static StringRef name() { return "NoOpModuleAnalysis"; }
 257 };
 258
 259 /// No-op CGSCC pass which does nothing.
 260 struct NoOpCGSCCPass {
 261   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
 262                         LazyCallGraph &, CGSCCUpdateResult &UR) {
 263     return PreservedAnalyses::all();
 264   }
 265   static StringRef name() { return "NoOpCGSCCPass"; }
 266 };
 267
 268 /// No-op CGSCC analysis.
 269 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
 270   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
 271   static AnalysisKey Key;
 272
 273 public:
 274   struct Result {};
 275   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
 276     return Result();
 277   }
 278   static StringRef name() { return "NoOpCGSCCAnalysis"; }
 279 };
 280
 281 /// No-op function pass which does nothing.
 282 struct NoOpFunctionPass {
 283   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
 284     return PreservedAnalyses::all();
 285   }
 286   static StringRef name() { return "NoOpFunctionPass"; }
 287 };
 288
 289 /// No-op function analysis.
 290 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
 291   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
 292   static AnalysisKey Key;
 293
 294 public:
 295   struct Result {};
 296   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
 297   static StringRef name() { return "NoOpFunctionAnalysis"; }
 298 };
 299
 300 /// No-op loop pass which does nothing.
 301 struct NoOpLoopPass {
 302   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
 303                         LoopStandardAnalysisResults &, LPMUpdater &) {
 304     return PreservedAnalyses::all();
 305   }
 306   static StringRef name() { return "NoOpLoopPass"; }
 307 };
 308
 309 /// No-op loop analysis.
 310 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
 311   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
 312   static AnalysisKey Key;
 313
 314 public:
 315   struct Result {};
 316   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
 317     return Result();
 318   }
 319   static StringRef name() { return "NoOpLoopAnalysis"; }
 320 };
 321
 322 AnalysisKey NoOpModuleAnalysis::Key;
 323 AnalysisKey NoOpCGSCCAnalysis::Key;
 324 AnalysisKey NoOpFunctionAnalysis::Key;
 325 AnalysisKey NoOpLoopAnalysis::Key;
 326
 327 } // End anonymous namespace.
 328
 329 void PassBuilder::invokePeepholeEPCallbacks(
 330     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
 331   for (auto &C : PeepholeEPCallbacks)
 332     C(FPM, Level);
 333 }
 334
 335 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
 336 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
 337   MAM.registerPass([&] { return CREATE_PASS; });
 338 #include "PassRegistry.def"
 339
 340   for (auto &C : ModuleAnalysisRegistrationCallbacks)
 341     C(MAM);
 342 }
 343
 344 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
 345 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
 346   CGAM.registerPass([&] { return CREATE_PASS; });
 347 #include "PassRegistry.def"
 348
 349   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
 350     C(CGAM);
 351 }
 352
 353 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
 354 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
 355   FAM.registerPass([&] { return CREATE_PASS; });
 356 #include "PassRegistry.def"
 357
 358   for (auto &C : FunctionAnalysisRegistrationCallbacks)
 359     C(FAM);
 360 }
 361
 362 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
 363 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 364   LAM.registerPass([&] { return CREATE_PASS; });
 365 #include "PassRegistry.def"
 366
 367   for (auto &C : LoopAnalysisRegistrationCallbacks)
 368     C(LAM);
 369 }
 370
 371 FunctionPassManager
 372 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 373                                                  ThinLTOPhase Phase,
 374                                                  bool DebugLogging) {
 375   assert(Level != O0 && "Must request optimizations!");
 376   FunctionPassManager FPM(DebugLogging);
 377
 378   // Form SSA out of local memory accesses after breaking apart aggregates into
 379   // scalars.
 380   FPM.addPass(SROA());
 381
 382   // Catch trivial redundancies
 383   FPM.addPass(EarlyCSEPass(EnableEarlyCSEMemSSA));
 384
 385   // Hoisting of scalars and load expressions.
 386   if (EnableGVNHoist)
 387     FPM.addPass(GVNHoistPass());
 388
 389   // Global value numbering based sinking.
 390   if (EnableGVNSink) {
 391     FPM.addPass(GVNSinkPass());
 392     FPM.addPass(SimplifyCFGPass());
 393   }
 394
 395   // Speculative execution if the target has divergent branches; otherwise nop.
 396   FPM.addPass(SpeculativeExecutionPass());
 397
 398   // Optimize based on known information about branches, and cleanup afterward.
 399   FPM.addPass(JumpThreadingPass());
 400   FPM.addPass(CorrelatedValuePropagationPass());
 401   FPM.addPass(SimplifyCFGPass());
 402   if (Level == O3)
 403     FPM.addPass(AggressiveInstCombinePass());
 404   FPM.addPass(InstCombinePass());
 405
 406   if (!isOptimizingForSize(Level))
 407     FPM.addPass(LibCallsShrinkWrapPass());
 408
 409   invokePeepholeEPCallbacks(FPM, Level);
 410
 411   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
 412   // using the size value profile. Don't perform this when optimizing for size.
 413   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
 414       !isOptimizingForSize(Level))
 415     FPM.addPass(PGOMemOPSizeOpt());
 416
 417   FPM.addPass(TailCallElimPass());
 418   FPM.addPass(SimplifyCFGPass());
 419
 420   // Form canonically associated expression trees, and simplify the trees using
 421   // basic mathematical properties. For example, this will form (nearly)
 422   // minimal multiplication trees.
 423   FPM.addPass(ReassociatePass());
 424
 425   // Add the primary loop simplification pipeline.
 426   // FIXME: Currently this is split into two loop pass pipelines because we run
 427   // some function passes in between them. These can and should be removed
 428   // and/or replaced by scheduling the loop pass equivalents in the correct
 429   // positions. But those equivalent passes aren't powerful enough yet.
 430   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
 431   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
 432   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
 433   // `LoopInstSimplify`.
 434   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
 435
 436   // Simplify the loop body. We do this initially to clean up after other loop
 437   // passes run, either when iterating on a loop or on inner loops with
 438   // implications on the outer loop.
 439   LPM1.addPass(LoopInstSimplifyPass());
 440   LPM1.addPass(LoopSimplifyCFGPass());
 441
 442   // Rotate Loop - disable header duplication at -Oz
 443   LPM1.addPass(LoopRotatePass(Level != Oz));
 444   LPM1.addPass(LICMPass());
 445   LPM1.addPass(SimpleLoopUnswitchPass());
 446   LPM2.addPass(IndVarSimplifyPass());
 447   LPM2.addPass(LoopIdiomRecognizePass());
 448
 449   for (auto &C : LateLoopOptimizationsEPCallbacks)
 450     C(LPM2, Level);
 451
 452   LPM2.addPass(LoopDeletionPass());
 453   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
 454   // because it changes IR to makes profile annotation in back compile
 455   // inaccurate.
 456   if (Phase != ThinLTOPhase::PreLink || !PGOOpt ||
 457       PGOOpt->Action != PGOOptions::SampleUse)
 458     LPM2.addPass(LoopFullUnrollPass(Level));
 459
 460   for (auto &C : LoopOptimizerEndEPCallbacks)
 461     C(LPM2, Level);
 462
 463   // We provide the opt remark emitter pass for LICM to use. We only need to do
 464   // this once as it is immutable.
 465   FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 466   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging));
 467   FPM.addPass(SimplifyCFGPass());
 468   FPM.addPass(InstCombinePass());
 469   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging));
 470
 471   // Eliminate redundancies.
 472   if (Level != O1) {
 473     // These passes add substantial compile time so skip them at O1.
 474     FPM.addPass(MergedLoadStoreMotionPass());
 475     if (RunNewGVN)
 476       FPM.addPass(NewGVNPass());
 477     else
 478       FPM.addPass(GVN());
 479   }
 480
 481   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
 482   FPM.addPass(MemCpyOptPass());
 483
 484   // Sparse conditional constant propagation.
 485   // FIXME: It isn't clear why we do this *after* loop passes rather than
 486   // before...
 487   FPM.addPass(SCCPPass());
 488
 489   // Delete dead bit computations (instcombine runs after to fold away the dead
 490   // computations, and then ADCE will run later to exploit any new DCE
 491   // opportunities that creates).
 492   FPM.addPass(BDCEPass());
 493
 494   // Run instcombine after redundancy and dead bit elimination to exploit
 495   // opportunities opened up by them.
 496   FPM.addPass(InstCombinePass());
 497   invokePeepholeEPCallbacks(FPM, Level);
 498
 499   // Re-consider control flow based optimizations after redundancy elimination,
 500   // redo DCE, etc.
 501   FPM.addPass(JumpThreadingPass());
 502   FPM.addPass(CorrelatedValuePropagationPass());
 503   FPM.addPass(DSEPass());
 504   FPM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging));
 505
 506   for (auto &C : ScalarOptimizerLateEPCallbacks)
 507     C(FPM, Level);
 508
 509   // Finally, do an expensive DCE pass to catch all the dead code exposed by
 510   // the simplifications and basic cleanup after all the simplifications.
 511   FPM.addPass(ADCEPass());
 512   FPM.addPass(SimplifyCFGPass());
 513   FPM.addPass(InstCombinePass());
 514   invokePeepholeEPCallbacks(FPM, Level);
 515
 516   if (EnableCHR && Level == O3 && PGOOpt &&
 517       (PGOOpt->Action == PGOOptions::IRUse ||
 518        PGOOpt->Action == PGOOptions::SampleUse))
 519     FPM.addPass(ControlHeightReductionPass());
 520
 521   return FPM;
 522 }
 523
 524 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
 525                                     PassBuilder::OptimizationLevel Level,
 526                                     bool RunProfileGen, bool IsCS,
 527                                     std::string ProfileFile,
 528                                     std::string ProfileRemappingFile) {
 529   // Generally running simplification passes and the inliner with an high
 530   // threshold results in smaller executables, but there may be cases where
 531   // the size grows, so let's be conservative here and skip this simplification
 532   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
 533   // IsCS is true).
 534   if (!isOptimizingForSize(Level) && !IsCS) {
 535     InlineParams IP;
 536
 537     // In the old pass manager, this is a cl::opt. Should still this be one?
 538     IP.DefaultThreshold = 75;
 539
 540     // FIXME: The hint threshold has the same value used by the regular inliner.
 541     // This should probably be lowered after performance testing.
 542     // FIXME: this comment is cargo culted from the old pass manager, revisit).
 543     IP.HintThreshold = 325;
 544
 545     CGSCCPassManager CGPipeline(DebugLogging);
 546
 547     CGPipeline.addPass(InlinerPass(IP));
 548
 549     FunctionPassManager FPM;
 550     FPM.addPass(SROA());
 551     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
 552     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
 553     FPM.addPass(InstCombinePass()); // Combine silly sequences.
 554     invokePeepholeEPCallbacks(FPM, Level);
 555
 556     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
 557
 558     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
 559   }
 560
 561   // Delete anything that is now dead to make sure that we don't instrument
 562   // dead code. Instrumentation can end up keeping dead code around and
 563   // dramatically increase code size.
 564   MPM.addPass(GlobalDCEPass());
 565
 566   if (RunProfileGen) {
 567     MPM.addPass(PGOInstrumentationGen(IsCS));
 568
 569     FunctionPassManager FPM;
 570     FPM.addPass(
 571         createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 572     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 573
 574     // Add the profile lowering pass.
 575     InstrProfOptions Options;
 576     if (!ProfileFile.empty())
 577       Options.InstrProfileOutput = ProfileFile;
 578     Options.DoCounterPromotion = true;
 579     Options.UseBFIInPromotion = IsCS;
 580     MPM.addPass(InstrProfiling(Options, IsCS));
 581   } else if (!ProfileFile.empty()) {
 582     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
 583     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 584     // RequireAnalysisPass for PSI before subsequent non-module passes.
 585     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 586   }
 587 }
 588
 589 static InlineParams
 590 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
 591   auto O3 = PassBuilder::O3;
 592   unsigned OptLevel = Level > O3 ? 2 : Level;
 593   unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
 594   return getInlineParams(OptLevel, SizeLevel);
 595 }
 596
 597 ModulePassManager
 598 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
 599                                                ThinLTOPhase Phase,
 600                                                bool DebugLogging) {
 601   ModulePassManager MPM(DebugLogging);
 602
 603   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
 604
 605   // In ThinLTO mode, when flattened profile is used, all the available
 606   // profile information will be annotated in PreLink phase so there is
 607   // no need to load the profile again in PostLink.
 608   bool LoadSampleProfile =
 609       HasSampleProfile &&
 610       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
 611
 612   // During the ThinLTO backend phase we perform early indirect call promotion
 613   // here, before globalopt. Otherwise imported available_externally functions
 614   // look unreferenced and are removed. If we are going to load the sample
 615   // profile then defer until later.
 616   // TODO: See if we can move later and consolidate with the location where
 617   // we perform ICP when we are loading a sample profile.
 618   // TODO: We pass HasSampleProfile (whether there was a sample profile file
 619   // passed to the compile) to the SamplePGO flag of ICP. This is used to
 620   // determine whether the new direct calls are annotated with prof metadata.
 621   // Ideally this should be determined from whether the IR is annotated with
 622   // sample profile, and not whether the a sample profile was provided on the
 623   // command line. E.g. for flattened profiles where we will not be reloading
 624   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
 625   // provide the sample profile file.
 626   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
 627     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
 628
 629   // Do basic inference of function attributes from known properties of system
 630   // libraries and other oracles.
 631   MPM.addPass(InferFunctionAttrsPass());
 632
 633   // Create an early function pass manager to cleanup the output of the
 634   // frontend.
 635   FunctionPassManager EarlyFPM(DebugLogging);
 636   EarlyFPM.addPass(SimplifyCFGPass());
 637   EarlyFPM.addPass(SROA());
 638   EarlyFPM.addPass(EarlyCSEPass());
 639   EarlyFPM.addPass(LowerExpectIntrinsicPass());
 640   if (Level == O3)
 641     EarlyFPM.addPass(CallSiteSplittingPass());
 642
 643   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
 644   // to convert bitcast to direct calls so that they can be inlined during the
 645   // profile annotation prepration step.
 646   // More details about SamplePGO design can be found in:
 647   // https://research.google.com/pubs/pub45290.html
 648   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
 649   if (LoadSampleProfile)
 650     EarlyFPM.addPass(InstCombinePass());
 651   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
 652
 653   if (LoadSampleProfile) {
 654     // Annotate sample profile right after early FPM to ensure freshness of
 655     // the debug info.
 656     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
 657                                         PGOOpt->ProfileRemappingFile,
 658                                         Phase == ThinLTOPhase::PreLink));
 659     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 660     // RequireAnalysisPass for PSI before subsequent non-module passes.
 661     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 662     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
 663     // for the profile annotation to be accurate in the ThinLTO backend.
 664     if (Phase != ThinLTOPhase::PreLink)
 665       // We perform early indirect call promotion here, before globalopt.
 666       // This is important for the ThinLTO backend phase because otherwise
 667       // imported available_externally functions look unreferenced and are
 668       // removed.
 669       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
 670                                            true /* SamplePGO */));
 671   }
 672
 673   // Interprocedural constant propagation now that basic cleanup has occurred
 674   // and prior to optimizing globals.
 675   // FIXME: This position in the pipeline hasn't been carefully considered in
 676   // years, it should be re-analyzed.
 677   MPM.addPass(IPSCCPPass());
 678
 679   // Attach metadata to indirect call sites indicating the set of functions
 680   // they may target at run-time. This should follow IPSCCP.
 681   MPM.addPass(CalledValuePropagationPass());
 682
 683   // Optimize globals to try and fold them into constants.
 684   MPM.addPass(GlobalOptPass());
 685
 686   // Promote any localized globals to SSA registers.
 687   // FIXME: Should this instead by a run of SROA?
 688   // FIXME: We should probably run instcombine and simplify-cfg afterward to
 689   // delete control flows that are dead once globals have been folded to
 690   // constants.
 691   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
 692
 693   // Remove any dead arguments exposed by cleanups and constand folding
 694   // globals.
 695   MPM.addPass(DeadArgumentEliminationPass());
 696
 697   // Create a small function pass pipeline to cleanup after all the global
 698   // optimizations.
 699   FunctionPassManager GlobalCleanupPM(DebugLogging);
 700   GlobalCleanupPM.addPass(InstCombinePass());
 701   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
 702
 703   GlobalCleanupPM.addPass(SimplifyCFGPass());
 704   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
 705
 706   // Add all the requested passes for instrumentation PGO, if requested.
 707   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 708       (PGOOpt->Action == PGOOptions::IRInstr ||
 709        PGOOpt->Action == PGOOptions::IRUse)) {
 710     addPGOInstrPasses(MPM, DebugLogging, Level,
 711                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
 712                       /* IsCS */ false, PGOOpt->ProfileFile,
 713                       PGOOpt->ProfileRemappingFile);
 714     MPM.addPass(PGOIndirectCallPromotion(false, false));
 715   }
 716   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 717       PGOOpt->CSAction == PGOOptions::CSIRInstr)
 718     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
 719
 720   // Synthesize function entry counts for non-PGO compilation.
 721   if (EnableSyntheticCounts && !PGOOpt)
 722     MPM.addPass(SyntheticCountsPropagation());
 723
 724   // Require the GlobalsAA analysis for the module so we can query it within
 725   // the CGSCC pipeline.
 726   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 727
 728   // Require the ProfileSummaryAnalysis for the module so we can query it within
 729   // the inliner pass.
 730   MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 731
 732   // Now begin the main postorder CGSCC pipeline.
 733   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
 734   // manager and trying to emulate its precise behavior. Much of this doesn't
 735   // make a lot of sense and we should revisit the core CGSCC structure.
 736   CGSCCPassManager MainCGPipeline(DebugLogging);
 737
 738   // Note: historically, the PruneEH pass was run first to deduce nounwind and
 739   // generally clean up exception handling overhead. It isn't clear this is
 740   // valuable as the inliner doesn't currently care whether it is inlining an
 741   // invoke or a call.
 742
 743   // Run the inliner first. The theory is that we are walking bottom-up and so
 744   // the callees have already been fully optimized, and we want to inline them
 745   // into the callers so that our optimizations can reflect that.
 746   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
 747   // because it makes profile annotation in the backend inaccurate.
 748   InlineParams IP = getInlineParamsFromOptLevel(Level);
 749   if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
 750       PGOOpt->Action == PGOOptions::SampleUse)
 751     IP.HotCallSiteThreshold = 0;
 752   MainCGPipeline.addPass(InlinerPass(IP));
 753
 754   // Now deduce any function attributes based in the current code.
 755   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
 756
 757   // When at O3 add argument promotion to the pass pipeline.
 758   // FIXME: It isn't at all clear why this should be limited to O3.
 759   if (Level == O3)
 760     MainCGPipeline.addPass(ArgumentPromotionPass());
 761
 762   // Lastly, add the core function simplification pipeline nested inside the
 763   // CGSCC walk.
 764   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
 765       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
 766
 767   for (auto &C : CGSCCOptimizerLateEPCallbacks)
 768     C(MainCGPipeline, Level);
 769
 770   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
 771   // to detect when we devirtualize indirect calls and iterate the SCC passes
 772   // in that case to try and catch knock-on inlining or function attrs
 773   // opportunities. Then we add it to the module pipeline by walking the SCCs
 774   // in postorder (or bottom-up).
 775   MPM.addPass(
 776       createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
 777           std::move(MainCGPipeline), MaxDevirtIterations)));
 778
 779   return MPM;
 780 }
 781
 782 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
 783     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
 784   ModulePassManager MPM(DebugLogging);
 785
 786   // Optimize globals now that the module is fully simplified.
 787   MPM.addPass(GlobalOptPass());
 788   MPM.addPass(GlobalDCEPass());
 789
 790   // Run partial inlining pass to partially inline functions that have
 791   // large bodies.
 792   if (RunPartialInlining)
 793     MPM.addPass(PartialInlinerPass());
 794
 795   // Remove avail extern fns and globals definitions since we aren't compiling
 796   // an object file for later LTO. For LTO we want to preserve these so they
 797   // are eligible for inlining at link-time. Note if they are unreferenced they
 798   // will be removed by GlobalDCE later, so this only impacts referenced
 799   // available externally globals. Eventually they will be suppressed during
 800   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
 801   // may make globals referenced by available external functions dead and saves
 802   // running remaining passes on the eliminated functions.
 803   MPM.addPass(EliminateAvailableExternallyPass());
 804
 805   if (EnableOrderFileInstrumentation)
 806     MPM.addPass(InstrOrderFilePass());
 807
 808   // Do RPO function attribute inference across the module to forward-propagate
 809   // attributes where applicable.
 810   // FIXME: Is this really an optimization rather than a canonicalization?
 811   MPM.addPass(ReversePostOrderFunctionAttrsPass());
 812
 813   // Do a post inline PGO instrumentation and use pass. This is a context
 814   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
 815   // cross-module inline has not been done yet. The context sensitive
 816   // instrumentation is after all the inlines are done.
 817   if (!LTOPreLink && PGOOpt) {
 818     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
 819       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
 820                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
 821                         PGOOpt->ProfileRemappingFile);
 822     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
 823       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
 824                         /* IsCS */ true, PGOOpt->ProfileFile,
 825                         PGOOpt->ProfileRemappingFile);
 826   }
 827
 828   // Re-require GloblasAA here prior to function passes. This is particularly
 829   // useful as the above will have inlined, DCE'ed, and function-attr
 830   // propagated everything. We should at this point have a reasonably minimal
 831   // and richly annotated call graph. By computing aliasing and mod/ref
 832   // information for all local globals here, the late loop passes and notably
 833   // the vectorizer will be able to use them to help recognize vectorizable
 834   // memory operations.
 835   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 836
 837   FunctionPassManager OptimizePM(DebugLogging);
 838   OptimizePM.addPass(Float2IntPass());
 839   // FIXME: We need to run some loop optimizations to re-rotate loops after
 840   // simplify-cfg and others undo their rotation.
 841
 842   // Optimize the loop execution. These passes operate on entire loop nests
 843   // rather than on each loop in an inside-out manner, and so they are actually
 844   // function passes.
 845
 846   for (auto &C : VectorizerStartEPCallbacks)
 847     C(OptimizePM, Level);
 848
 849   // First rotate loops that may have been un-rotated by prior passes.
 850   OptimizePM.addPass(
 851       createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 852
 853   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
 854   // into separate loop that would otherwise inhibit vectorization.  This is
 855   // currently only performed for loops marked with the metadata
 856   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
 857   OptimizePM.addPass(LoopDistributePass());
 858
 859   // Now run the core loop vectorizer.
 860   OptimizePM.addPass(LoopVectorizePass());
 861
 862   // Eliminate loads by forwarding stores from the previous iteration to loads
 863   // of the current iteration.
 864   OptimizePM.addPass(LoopLoadEliminationPass());
 865
 866   // Cleanup after the loop optimization passes.
 867   OptimizePM.addPass(InstCombinePass());
 868
 869   // Now that we've formed fast to execute loop structures, we do further
 870   // optimizations. These are run afterward as they might block doing complex
 871   // analyses and transforms such as what are needed for loop vectorization.
 872
 873   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
 874   // GVN, loop transforms, and others have already run, so it's now better to
 875   // convert to more optimized IR using more aggressive simplify CFG options.
 876   // The extra sinking transform can create larger basic blocks, so do this
 877   // before SLP vectorization.
 878   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
 879                                      forwardSwitchCondToPhi(true).
 880                                      convertSwitchToLookupTable(true).
 881                                      needCanonicalLoops(false).
 882                                      sinkCommonInsts(true)));
 883
 884   // Optimize parallel scalar instruction chains into SIMD instructions.
 885   OptimizePM.addPass(SLPVectorizerPass());
 886
 887   OptimizePM.addPass(InstCombinePass());
 888
 889   // Unroll small loops to hide loop backedge latency and saturate any parallel
 890   // execution resources of an out-of-order processor. We also then need to
 891   // clean up redundancies and loop invariant code.
 892   // FIXME: It would be really good to use a loop-integrated instruction
 893   // combiner for cleanup here so that the unrolling and LICM can be pipelined
 894   // across the loop nests.
 895   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
 896   if (EnableUnrollAndJam) {
 897     OptimizePM.addPass(
 898         createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level)));
 899   }
 900   OptimizePM.addPass(LoopUnrollPass(LoopUnrollOptions(Level)));
 901   OptimizePM.addPass(WarnMissedTransformationsPass());
 902   OptimizePM.addPass(InstCombinePass());
 903   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 904   OptimizePM.addPass(createFunctionToLoopPassAdaptor(LICMPass(), DebugLogging));
 905
 906   // Now that we've vectorized and unrolled loops, we may have more refined
 907   // alignment information, try to re-derive it here.
 908   OptimizePM.addPass(AlignmentFromAssumptionsPass());
 909
 910   // Split out cold code. Splitting is done late to avoid hiding context from
 911   // other optimizations and inadvertently regressing performance. The tradeoff
 912   // is that this has a higher code size cost than splitting early.
 913   if (EnableHotColdSplit && !LTOPreLink)
 914     MPM.addPass(HotColdSplittingPass());
 915
 916   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
 917   // canonicalization pass that enables other optimizations. As a result,
 918   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
 919   // result too early.
 920   OptimizePM.addPass(LoopSinkPass());
 921
 922   // And finally clean up LCSSA form before generating code.
 923   OptimizePM.addPass(InstSimplifyPass());
 924
 925   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
 926   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
 927   // flattening of blocks.
 928   OptimizePM.addPass(DivRemPairsPass());
 929
 930   // LoopSink (and other loop passes since the last simplifyCFG) might have
 931   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
 932   OptimizePM.addPass(SimplifyCFGPass());
 933
 934   // Optimize PHIs by speculating around them when profitable. Note that this
 935   // pass needs to be run after any PRE or similar pass as it is essentially
 936   // inserting redudnancies into the progrem. This even includes SimplifyCFG.
 937   OptimizePM.addPass(SpeculateAroundPHIsPass());
 938
 939   for (auto &C : OptimizerLastEPCallbacks)
 940     C(OptimizePM, Level);
 941
 942   // Add the core optimizing pipeline.
 943   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
 944
 945   MPM.addPass(CGProfilePass());
 946
 947   // Now we need to do some global optimization transforms.
 948   // FIXME: It would seem like these should come first in the optimization
 949   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
 950   // ordering here.
 951   MPM.addPass(GlobalDCEPass());
 952   MPM.addPass(ConstantMergePass());
 953
 954   return MPM;
 955 }
 956
 957 ModulePassManager
 958 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
 959                                            bool DebugLogging, bool LTOPreLink) {
 960   assert(Level != O0 && "Must request optimizations for the default pipeline!");
 961
 962   ModulePassManager MPM(DebugLogging);
 963
 964   // Force any function attributes we want the rest of the pipeline to observe.
 965   MPM.addPass(ForceFunctionAttrsPass());
 966
 967   // Apply module pipeline start EP callback.
 968   for (auto &C : PipelineStartEPCallbacks)
 969     C(MPM);
 970
 971   if (PGOOpt && PGOOpt->SamplePGOSupport)
 972     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 973
 974   // Add the core simplification pipeline.
 975   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
 976                                                 DebugLogging));
 977
 978   // Now add the optimization pipeline.
 979   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
 980
 981   return MPM;
 982 }
 983
 984 ModulePassManager
 985 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
 986                                                 bool DebugLogging) {
 987   assert(Level != O0 && "Must request optimizations for the default pipeline!");
 988
 989   ModulePassManager MPM(DebugLogging);
 990
 991   // Force any function attributes we want the rest of the pipeline to observe.
 992   MPM.addPass(ForceFunctionAttrsPass());
 993
 994   if (PGOOpt && PGOOpt->SamplePGOSupport)
 995     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 996
 997   // Apply module pipeline start EP callback.
 998   for (auto &C : PipelineStartEPCallbacks)
 999     C(MPM);
1000
1001   // If we are planning to perform ThinLTO later, we don't bloat the code with
1002   // unrolling/vectorization/... now. Just simplify the module as much as we
1003   // can.
1004   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1005                                                 DebugLogging));
1006
1007   // Run partial inlining pass to partially inline functions that have
1008   // large bodies.
1009   // FIXME: It isn't clear whether this is really the right place to run this
1010   // in ThinLTO. Because there is another canonicalization and simplification
1011   // phase that will run after the thin link, running this here ends up with
1012   // less information than will be available later and it may grow functions in
1013   // ways that aren't beneficial.
1014   if (RunPartialInlining)
1015     MPM.addPass(PartialInlinerPass());
1016
1017   // Reduce the size of the IR as much as possible.
1018   MPM.addPass(GlobalOptPass());
1019
1020   return MPM;
1021 }
1022
1023 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1024     OptimizationLevel Level, bool DebugLogging,
1025     const ModuleSummaryIndex *ImportSummary) {
1026   ModulePassManager MPM(DebugLogging);
1027
1028   if (ImportSummary) {
1029     // These passes import type identifier resolutions for whole-program
1030     // devirtualization and CFI. They must run early because other passes may
1031     // disturb the specific instruction patterns that these passes look for,
1032     // creating dependencies on resolutions that may not appear in the summary.
1033     //
1034     // For example, GVN may transform the pattern assume(type.test) appearing in
1035     // two basic blocks into assume(phi(type.test, type.test)), which would
1036     // transform a dependency on a WPD resolution into a dependency on a type
1037     // identifier resolution for CFI.
1038     //
1039     // Also, WPD has access to more precise information than ICP and can
1040     // devirtualize more effectively, so it should operate on the IR first.
1041     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1042     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1043   }
1044
1045   // Force any function attributes we want the rest of the pipeline to observe.
1046   MPM.addPass(ForceFunctionAttrsPass());
1047
1048   // Add the core simplification pipeline.
1049   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1050                                                 DebugLogging));
1051
1052   // Now add the optimization pipeline.
1053   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1054
1055   return MPM;
1056 }
1057
1058 ModulePassManager
1059 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1060                                             bool DebugLogging) {
1061   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1062   // FIXME: We should use a customized pre-link pipeline!
1063   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1064                                        /* LTOPreLink */true);
1065 }
1066
1067 ModulePassManager
1068 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1069                                      ModuleSummaryIndex *ExportSummary) {
1070   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1071   ModulePassManager MPM(DebugLogging);
1072
1073   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1074     // Load sample profile before running the LTO optimization pipeline.
1075     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1076                                         PGOOpt->ProfileRemappingFile,
1077                                         false /* ThinLTOPhase::PreLink */));
1078     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1079     // RequireAnalysisPass for PSI before subsequent non-module passes.
1080     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1081   }
1082
1083   // Remove unused virtual tables to improve the quality of code generated by
1084   // whole-program devirtualization and bitset lowering.
1085   MPM.addPass(GlobalDCEPass());
1086
1087   // Force any function attributes we want the rest of the pipeline to observe.
1088   MPM.addPass(ForceFunctionAttrsPass());
1089
1090   // Do basic inference of function attributes from known properties of system
1091   // libraries and other oracles.
1092   MPM.addPass(InferFunctionAttrsPass());
1093
1094   if (Level > 1) {
1095     FunctionPassManager EarlyFPM(DebugLogging);
1096     EarlyFPM.addPass(CallSiteSplittingPass());
1097     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1098
1099     // Indirect call promotion. This should promote all the targets that are
1100     // left by the earlier promotion pass that promotes intra-module targets.
1101     // This two-step promotion is to save the compile time. For LTO, it should
1102     // produce the same result as if we only do promotion here.
1103     MPM.addPass(PGOIndirectCallPromotion(
1104         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1105     // Propagate constants at call sites into the functions they call.  This
1106     // opens opportunities for globalopt (and inlining) by substituting function
1107     // pointers passed as arguments to direct uses of functions.
1108    MPM.addPass(IPSCCPPass());
1109
1110    // Attach metadata to indirect call sites indicating the set of functions
1111    // they may target at run-time. This should follow IPSCCP.
1112    MPM.addPass(CalledValuePropagationPass());
1113   }
1114
1115   // Now deduce any function attributes based in the current code.
1116   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1117               PostOrderFunctionAttrsPass()));
1118
1119   // Do RPO function attribute inference across the module to forward-propagate
1120   // attributes where applicable.
1121   // FIXME: Is this really an optimization rather than a canonicalization?
1122   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1123
1124   // Use in-range annotations on GEP indices to split globals where beneficial.
1125   MPM.addPass(GlobalSplitPass());
1126
1127   // Run whole program optimization of virtual call when the list of callees
1128   // is fixed.
1129   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1130
1131   // Stop here at -O1.
1132   if (Level == 1) {
1133     // The LowerTypeTestsPass needs to run to lower type metadata and the
1134     // type.test intrinsics. The pass does nothing if CFI is disabled.
1135     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1136     return MPM;
1137   }
1138
1139   // Optimize globals to try and fold them into constants.
1140   MPM.addPass(GlobalOptPass());
1141
1142   // Promote any localized globals to SSA registers.
1143   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1144
1145   // Linking modules together can lead to duplicate global constant, only
1146   // keep one copy of each constant.
1147   MPM.addPass(ConstantMergePass());
1148
1149   // Remove unused arguments from functions.
1150   MPM.addPass(DeadArgumentEliminationPass());
1151
1152   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1153   // simplification opportunities, and both can propagate functions through
1154   // function pointers.  When this happens, we often have to resolve varargs
1155   // calls, etc, so let instcombine do this.
1156   FunctionPassManager PeepholeFPM(DebugLogging);
1157   if (Level == O3)
1158     PeepholeFPM.addPass(AggressiveInstCombinePass());
1159   PeepholeFPM.addPass(InstCombinePass());
1160   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1161
1162   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1163
1164   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1165   // generally clean up exception handling overhead. It isn't clear this is
1166   // valuable as the inliner doesn't currently care whether it is inlining an
1167   // invoke or a call.
1168   // Run the inliner now.
1169   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1170       InlinerPass(getInlineParamsFromOptLevel(Level))));
1171
1172   // Optimize globals again after we ran the inliner.
1173   MPM.addPass(GlobalOptPass());
1174
1175   // Garbage collect dead functions.
1176   // FIXME: Add ArgumentPromotion pass after once it's ported.
1177   MPM.addPass(GlobalDCEPass());
1178
1179   FunctionPassManager FPM(DebugLogging);
1180   // The IPO Passes may leave cruft around. Clean up after them.
1181   FPM.addPass(InstCombinePass());
1182   invokePeepholeEPCallbacks(FPM, Level);
1183
1184   FPM.addPass(JumpThreadingPass());
1185
1186   // Do a post inline PGO instrumentation and use pass. This is a context
1187   // sensitive PGO pass.
1188   if (PGOOpt) {
1189     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1190       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1191                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1192                         PGOOpt->ProfileRemappingFile);
1193     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1194       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1195                         /* IsCS */ true, PGOOpt->ProfileFile,
1196                         PGOOpt->ProfileRemappingFile);
1197   }
1198
1199   // Break up allocas
1200   FPM.addPass(SROA());
1201
1202   // LTO provides additional opportunities for tailcall elimination due to
1203   // link-time inlining, and visibility of nocapture attribute.
1204   FPM.addPass(TailCallElimPass());
1205
1206   // Run a few AA driver optimizations here and now to cleanup the code.
1207   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1208
1209   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1210               PostOrderFunctionAttrsPass()));
1211   // FIXME: here we run IP alias analysis in the legacy PM.
1212
1213   FunctionPassManager MainFPM;
1214
1215   // FIXME: once we fix LoopPass Manager, add LICM here.
1216   // FIXME: once we provide support for enabling MLSM, add it here.
1217   // FIXME: once we provide support for enabling NewGVN, add it here.
1218   if (RunNewGVN)
1219     MainFPM.addPass(NewGVNPass());
1220   else
1221     MainFPM.addPass(GVN());
1222
1223   // Remove dead memcpy()'s.
1224   MainFPM.addPass(MemCpyOptPass());
1225
1226   // Nuke dead stores.
1227   MainFPM.addPass(DSEPass());
1228
1229   // FIXME: at this point, we run a bunch of loop passes:
1230   // indVarSimplify, loopDeletion, loopInterchange, loopUnrool,
1231   // loopVectorize. Enable them once the remaining issue with LPM
1232   // are sorted out.
1233
1234   MainFPM.addPass(InstCombinePass());
1235   MainFPM.addPass(SimplifyCFGPass());
1236   MainFPM.addPass(SCCPPass());
1237   MainFPM.addPass(InstCombinePass());
1238   MainFPM.addPass(BDCEPass());
1239
1240   // FIXME: We may want to run SLPVectorizer here.
1241   // After vectorization, assume intrinsics may tell us more
1242   // about pointer alignments.
1243 #if 0
1244   MainFPM.add(AlignmentFromAssumptionsPass());
1245 #endif
1246
1247   // FIXME: Conditionally run LoadCombine here, after it's ported
1248   // (in case we still have this pass, given its questionable usefulness).
1249
1250   MainFPM.addPass(InstCombinePass());
1251   invokePeepholeEPCallbacks(MainFPM, Level);
1252   MainFPM.addPass(JumpThreadingPass());
1253   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1254
1255   // Create a function that performs CFI checks for cross-DSO calls with
1256   // targets in the current module.
1257   MPM.addPass(CrossDSOCFIPass());
1258
1259   // Lower type metadata and the type.test intrinsic. This pass supports
1260   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1261   // to be run at link time if CFI is enabled. This pass does nothing if
1262   // CFI is disabled.
1263   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1264
1265   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1266   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1267   if (EnableHotColdSplit)
1268     MPM.addPass(HotColdSplittingPass());
1269
1270   // Add late LTO optimization passes.
1271   // Delete basic blocks, which optimization passes may have killed.
1272   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1273
1274   // Drop bodies of available eternally objects to improve GlobalDCE.
1275   MPM.addPass(EliminateAvailableExternallyPass());
1276
1277   // Now that we have optimized the program, discard unreachable functions.
1278   MPM.addPass(GlobalDCEPass());
1279
1280   // FIXME: Enable MergeFuncs, conditionally, after ported, maybe.
1281   return MPM;
1282 }
1283
1284 AAManager PassBuilder::buildDefaultAAPipeline() {
1285   AAManager AA;
1286
1287   // The order in which these are registered determines their priority when
1288   // being queried.
1289
1290   // First we register the basic alias analysis that provides the majority of
1291   // per-function local AA logic. This is a stateless, on-demand local set of
1292   // AA techniques.
1293   AA.registerFunctionAnalysis<BasicAA>();
1294
1295   // Next we query fast, specialized alias analyses that wrap IR-embedded
1296   // information about aliasing.
1297   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1298   AA.registerFunctionAnalysis<TypeBasedAA>();
1299
1300   // Add support for querying global aliasing information when available.
1301   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1302   // analysis, all that the `AAManager` can do is query for any *cached*
1303   // results from `GlobalsAA` through a readonly proxy.
1304   AA.registerModuleAnalysis<GlobalsAA>();
1305
1306   return AA;
1307 }
1308
1309 static Optional<int> parseRepeatPassName(StringRef Name) {
1310   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1311     return None;
1312   int Count;
1313   if (Name.getAsInteger(0, Count) || Count <= 0)
1314     return None;
1315   return Count;
1316 }
1317
1318 static Optional<int> parseDevirtPassName(StringRef Name) {
1319   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1320     return None;
1321   int Count;
1322   if (Name.getAsInteger(0, Count) || Count <= 0)
1323     return None;
1324   return Count;
1325 }
1326
1327 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1328   if (!Name.consume_front(PassName))
1329     return false;
1330   // normal pass name w/o parameters == default parameters
1331   if (Name.empty())
1332     return true;
1333   return Name.startswith("<") && Name.endswith(">");
1334 }
1335
1336 namespace {
1337
1338 /// This performs customized parsing of pass name with parameters.
1339 ///
1340 /// We do not need parametrization of passes in textual pipeline very often,
1341 /// yet on a rare occasion ability to specify parameters right there can be
1342 /// useful.
1343 ///
1344 /// \p Name - parameterized specification of a pass from a textual pipeline
1345 /// is a string in a form of :
1346 ///      PassName '<' parameter-list '>'
1347 ///
1348 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1349 /// It takes a string-ref of parameters and returns either StringError or a
1350 /// parameter list in a form of a custom parameters type, all wrapped into
1351 /// Expected<> template class.
1352 ///
1353 template <typename ParametersParseCallableT>
1354 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1355                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1356   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1357
1358   StringRef Params = Name;
1359   if (!Params.consume_front(PassName)) {
1360     assert(false &&
1361            "unable to strip pass name from parametrized pass specification");
1362   }
1363   if (Params.empty())
1364     return ParametersT{};
1365   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1366     assert(false && "invalid format for parametrized pass name");
1367   }
1368
1369   Expected<ParametersT> Result = Parser(Params);
1370   assert((Result || Result.template errorIsA<StringError>()) &&
1371          "Pass parameter parser can only return StringErrors.");
1372   return std::move(Result);
1373 }
1374
1375 /// Parser of parameters for LoopUnroll pass.
1376 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1377   LoopUnrollOptions UnrollOpts;
1378   while (!Params.empty()) {
1379     StringRef ParamName;
1380     std::tie(ParamName, Params) = Params.split(';');
1381     int OptLevel = StringSwitch<int>(ParamName)
1382                        .Case("O0", 0)
1383                        .Case("O1", 1)
1384                        .Case("O2", 2)
1385                        .Case("O3", 3)
1386                        .Default(-1);
1387     if (OptLevel >= 0) {
1388       UnrollOpts.setOptLevel(OptLevel);
1389       continue;
1390     }
1391
1392     bool Enable = !ParamName.consume_front("no-");
1393     if (ParamName == "partial") {
1394       UnrollOpts.setPartial(Enable);
1395     } else if (ParamName == "peeling") {
1396       UnrollOpts.setPeeling(Enable);
1397     } else if (ParamName == "runtime") {
1398       UnrollOpts.setRuntime(Enable);
1399     } else if (ParamName == "upperbound") {
1400       UnrollOpts.setUpperBound(Enable);
1401     } else {
1402       return make_error<StringError>(
1403           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1404           inconvertibleErrorCode());
1405     }
1406   }
1407   return UnrollOpts;
1408 }
1409
1410 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1411   MemorySanitizerOptions Result;
1412   while (!Params.empty()) {
1413     StringRef ParamName;
1414     std::tie(ParamName, Params) = Params.split(';');
1415
1416     if (ParamName == "recover") {
1417       Result.Recover = true;
1418     } else if (ParamName == "kernel") {
1419       Result.Kernel = true;
1420     } else if (ParamName.consume_front("track-origins=")) {
1421       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1422         return make_error<StringError>(
1423             formatv("invalid argument to MemorySanitizer pass track-origins "
1424                     "parameter: '{0}' ",
1425                     ParamName)
1426                 .str(),
1427             inconvertibleErrorCode());
1428     } else {
1429       return make_error<StringError>(
1430           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1431               .str(),
1432           inconvertibleErrorCode());
1433     }
1434   }
1435   return Result;
1436 }
1437
1438 /// Parser of parameters for SimplifyCFG pass.
1439 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1440   SimplifyCFGOptions Result;
1441   while (!Params.empty()) {
1442     StringRef ParamName;
1443     std::tie(ParamName, Params) = Params.split(';');
1444
1445     bool Enable = !ParamName.consume_front("no-");
1446     if (ParamName == "forward-switch-cond") {
1447       Result.forwardSwitchCondToPhi(Enable);
1448     } else if (ParamName == "switch-to-lookup") {
1449       Result.convertSwitchToLookupTable(Enable);
1450     } else if (ParamName == "keep-loops") {
1451       Result.needCanonicalLoops(Enable);
1452     } else if (ParamName == "sink-common-insts") {
1453       Result.sinkCommonInsts(Enable);
1454     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1455       APInt BonusInstThreshold;
1456       if (ParamName.getAsInteger(0, BonusInstThreshold))
1457         return make_error<StringError>(
1458             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1459                     "parameter: '{0}' ",
1460                     ParamName).str(),
1461             inconvertibleErrorCode());
1462       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1463     } else {
1464       return make_error<StringError>(
1465           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1466           inconvertibleErrorCode());
1467     }
1468   }
1469   return Result;
1470 }
1471
1472 /// Parser of parameters for LoopVectorize pass.
1473 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1474   LoopVectorizeOptions Opts;
1475   while (!Params.empty()) {
1476     StringRef ParamName;
1477     std::tie(ParamName, Params) = Params.split(';');
1478
1479     bool Enable = !ParamName.consume_front("no-");
1480     if (ParamName == "interleave-forced-only") {
1481       Opts.setInterleaveOnlyWhenForced(Enable);
1482     } else if (ParamName == "vectorize-forced-only") {
1483       Opts.setVectorizeOnlyWhenForced(Enable);
1484     } else {
1485       return make_error<StringError>(
1486           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1487           inconvertibleErrorCode());
1488     }
1489   }
1490   return Opts;
1491 }
1492
1493 } // namespace
1494
1495 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1496 /// alias.
1497 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1498   return Name.startswith("default") || Name.startswith("thinlto") ||
1499          Name.startswith("lto");
1500 }
1501
1502 /// Tests whether registered callbacks will accept a given pass name.
1503 ///
1504 /// When parsing a pipeline text, the type of the outermost pipeline may be
1505 /// omitted, in which case the type is automatically determined from the first
1506 /// pass name in the text. This may be a name that is handled through one of the
1507 /// callbacks. We check this through the oridinary parsing callbacks by setting
1508 /// up a dummy PassManager in order to not force the client to also handle this
1509 /// type of query.
1510 template <typename PassManagerT, typename CallbacksT>
1511 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1512   if (!Callbacks.empty()) {
1513     PassManagerT DummyPM;
1514     for (auto &CB : Callbacks)
1515       if (CB(Name, DummyPM, {}))
1516         return true;
1517   }
1518   return false;
1519 }
1520
1521 template <typename CallbacksT>
1522 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1523   // Manually handle aliases for pre-configured pipeline fragments.
1524   if (startsWithDefaultPipelineAliasPrefix(Name))
1525     return DefaultAliasRegex.match(Name);
1526
1527   // Explicitly handle pass manager names.
1528   if (Name == "module")
1529     return true;
1530   if (Name == "cgscc")
1531     return true;
1532   if (Name == "function")
1533     return true;
1534
1535   // Explicitly handle custom-parsed pass names.
1536   if (parseRepeatPassName(Name))
1537     return true;
1538
1539 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1540   if (Name == NAME)                                                            \
1541     return true;
1542 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1543   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1544     return true;
1545 #include "PassRegistry.def"
1546
1547   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1548 }
1549
1550 template <typename CallbacksT>
1551 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
1552   // Explicitly handle pass manager names.
1553   if (Name == "cgscc")
1554     return true;
1555   if (Name == "function")
1556     return true;
1557
1558   // Explicitly handle custom-parsed pass names.
1559   if (parseRepeatPassName(Name))
1560     return true;
1561   if (parseDevirtPassName(Name))
1562     return true;
1563
1564 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1565   if (Name == NAME)                                                            \
1566     return true;
1567 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1568   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1569     return true;
1570 #include "PassRegistry.def"
1571
1572   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
1573 }
1574
1575 template <typename CallbacksT>
1576 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
1577   // Explicitly handle pass manager names.
1578   if (Name == "function")
1579     return true;
1580   if (Name == "loop")
1581     return true;
1582
1583   // Explicitly handle custom-parsed pass names.
1584   if (parseRepeatPassName(Name))
1585     return true;
1586
1587 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1588   if (Name == NAME)                                                            \
1589     return true;
1590 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1591   if (checkParametrizedPassName(Name, NAME))                                   \
1592     return true;
1593 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1594   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1595     return true;
1596 #include "PassRegistry.def"
1597
1598   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
1599 }
1600
1601 template <typename CallbacksT>
1602 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
1603   // Explicitly handle pass manager names.
1604   if (Name == "loop")
1605     return true;
1606
1607   // Explicitly handle custom-parsed pass names.
1608   if (parseRepeatPassName(Name))
1609     return true;
1610
1611 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
1612   if (Name == NAME)                                                            \
1613     return true;
1614 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
1615   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1616     return true;
1617 #include "PassRegistry.def"
1618
1619   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
1620 }
1621
1622 Optional<std::vector<PassBuilder::PipelineElement>>
1623 PassBuilder::parsePipelineText(StringRef Text) {
1624   std::vector<PipelineElement> ResultPipeline;
1625
1626   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1627       &ResultPipeline};
1628   for (;;) {
1629     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1630     size_t Pos = Text.find_first_of(",()");
1631     Pipeline.push_back({Text.substr(0, Pos), {}});
1632
1633     // If we have a single terminating name, we're done.
1634     if (Pos == Text.npos)
1635       break;
1636
1637     char Sep = Text[Pos];
1638     Text = Text.substr(Pos + 1);
1639     if (Sep == ',')
1640       // Just a name ending in a comma, continue.
1641       continue;
1642
1643     if (Sep == '(') {
1644       // Push the inner pipeline onto the stack to continue processing.
1645       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1646       continue;
1647     }
1648
1649     assert(Sep == ')' && "Bogus separator!");
1650     // When handling the close parenthesis, we greedily consume them to avoid
1651     // empty strings in the pipeline.
1652     do {
1653       // If we try to pop the outer pipeline we have unbalanced parentheses.
1654       if (PipelineStack.size() == 1)
1655         return None;
1656
1657       PipelineStack.pop_back();
1658     } while (Text.consume_front(")"));
1659
1660     // Check if we've finished parsing.
1661     if (Text.empty())
1662       break;
1663
1664     // Otherwise, the end of an inner pipeline always has to be followed by
1665     // a comma, and then we can continue.
1666     if (!Text.consume_front(","))
1667       return None;
1668   }
1669
1670   if (PipelineStack.size() > 1)
1671     // Unbalanced paretheses.
1672     return None;
1673
1674   assert(PipelineStack.back() == &ResultPipeline &&
1675          "Wrong pipeline at the bottom of the stack!");
1676   return {std::move(ResultPipeline)};
1677 }
1678
1679 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1680                                    const PipelineElement &E,
1681                                    bool VerifyEachPass, bool DebugLogging) {
1682   auto &Name = E.Name;
1683   auto &InnerPipeline = E.InnerPipeline;
1684
1685   // First handle complex passes like the pass managers which carry pipelines.
1686   if (!InnerPipeline.empty()) {
1687     if (Name == "module") {
1688       ModulePassManager NestedMPM(DebugLogging);
1689       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1690                                              VerifyEachPass, DebugLogging))
1691         return Err;
1692       MPM.addPass(std::move(NestedMPM));
1693       return Error::success();
1694     }
1695     if (Name == "cgscc") {
1696       CGSCCPassManager CGPM(DebugLogging);
1697       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1698                                             DebugLogging))
1699         return Err;
1700       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1701       return Error::success();
1702     }
1703     if (Name == "function") {
1704       FunctionPassManager FPM(DebugLogging);
1705       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1706                                                VerifyEachPass, DebugLogging))
1707         return Err;
1708       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1709       return Error::success();
1710     }
1711     if (auto Count = parseRepeatPassName(Name)) {
1712       ModulePassManager NestedMPM(DebugLogging);
1713       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1714                                              VerifyEachPass, DebugLogging))
1715         return Err;
1716       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
1717       return Error::success();
1718     }
1719
1720     for (auto &C : ModulePipelineParsingCallbacks)
1721       if (C(Name, MPM, InnerPipeline))
1722         return Error::success();
1723
1724     // Normal passes can't have pipelines.
1725     return make_error<StringError>(
1726         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1727         inconvertibleErrorCode());
1728     ;
1729   }
1730
1731   // Manually handle aliases for pre-configured pipeline fragments.
1732   if (startsWithDefaultPipelineAliasPrefix(Name)) {
1733     SmallVector<StringRef, 3> Matches;
1734     if (!DefaultAliasRegex.match(Name, &Matches))
1735       return make_error<StringError>(
1736           formatv("unknown default pipeline alias '{0}'", Name).str(),
1737           inconvertibleErrorCode());
1738
1739     assert(Matches.size() == 3 && "Must capture two matched strings!");
1740
1741     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
1742                               .Case("O0", O0)
1743                               .Case("O1", O1)
1744                               .Case("O2", O2)
1745                               .Case("O3", O3)
1746                               .Case("Os", Os)
1747                               .Case("Oz", Oz);
1748     if (L == O0)
1749       // At O0 we do nothing at all!
1750       return Error::success();
1751
1752     if (Matches[1] == "default") {
1753       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
1754     } else if (Matches[1] == "thinlto-pre-link") {
1755       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1756     } else if (Matches[1] == "thinlto") {
1757       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
1758     } else if (Matches[1] == "lto-pre-link") {
1759       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
1760     } else {
1761       assert(Matches[1] == "lto" && "Not one of the matched options!");
1762       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
1763     }
1764     return Error::success();
1765   }
1766
1767   // Finally expand the basic registered passes from the .inc file.
1768 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1769   if (Name == NAME) {                                                          \
1770     MPM.addPass(CREATE_PASS);                                                  \
1771     return Error::success();                                                   \
1772   }
1773 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1774   if (Name == "require<" NAME ">") {                                           \
1775     MPM.addPass(                                                               \
1776         RequireAnalysisPass<                                                   \
1777             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
1778     return Error::success();                                                   \
1779   }                                                                            \
1780   if (Name == "invalidate<" NAME ">") {                                        \
1781     MPM.addPass(InvalidateAnalysisPass<                                        \
1782                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
1783     return Error::success();                                                   \
1784   }
1785 #include "PassRegistry.def"
1786
1787   for (auto &C : ModulePipelineParsingCallbacks)
1788     if (C(Name, MPM, InnerPipeline))
1789       return Error::success();
1790   return make_error<StringError>(
1791       formatv("unknown module pass '{0}'", Name).str(),
1792       inconvertibleErrorCode());
1793 }
1794
1795 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1796                                   const PipelineElement &E, bool VerifyEachPass,
1797                                   bool DebugLogging) {
1798   auto &Name = E.Name;
1799   auto &InnerPipeline = E.InnerPipeline;
1800
1801   // First handle complex passes like the pass managers which carry pipelines.
1802   if (!InnerPipeline.empty()) {
1803     if (Name == "cgscc") {
1804       CGSCCPassManager NestedCGPM(DebugLogging);
1805       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1806                                             VerifyEachPass, DebugLogging))
1807         return Err;
1808       // Add the nested pass manager with the appropriate adaptor.
1809       CGPM.addPass(std::move(NestedCGPM));
1810       return Error::success();
1811     }
1812     if (Name == "function") {
1813       FunctionPassManager FPM(DebugLogging);
1814       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1815                                                VerifyEachPass, DebugLogging))
1816         return Err;
1817       // Add the nested pass manager with the appropriate adaptor.
1818       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
1819       return Error::success();
1820     }
1821     if (auto Count = parseRepeatPassName(Name)) {
1822       CGSCCPassManager NestedCGPM(DebugLogging);
1823       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1824                                             VerifyEachPass, DebugLogging))
1825         return Err;
1826       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
1827       return Error::success();
1828     }
1829     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1830       CGSCCPassManager NestedCGPM(DebugLogging);
1831       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1832                                             VerifyEachPass, DebugLogging))
1833         return Err;
1834       CGPM.addPass(
1835           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
1836       return Error::success();
1837     }
1838
1839     for (auto &C : CGSCCPipelineParsingCallbacks)
1840       if (C(Name, CGPM, InnerPipeline))
1841         return Error::success();
1842
1843     // Normal passes can't have pipelines.
1844     return make_error<StringError>(
1845         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
1846         inconvertibleErrorCode());
1847   }
1848
1849 // Now expand the basic registered passes from the .inc file.
1850 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1851   if (Name == NAME) {                                                          \
1852     CGPM.addPass(CREATE_PASS);                                                 \
1853     return Error::success();                                                   \
1854   }
1855 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1856   if (Name == "require<" NAME ">") {                                           \
1857     CGPM.addPass(RequireAnalysisPass<                                          \
1858                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
1859                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
1860                  CGSCCUpdateResult &>());                                      \
1861     return Error::success();                                                   \
1862   }                                                                            \
1863   if (Name == "invalidate<" NAME ">") {                                        \
1864     CGPM.addPass(InvalidateAnalysisPass<                                       \
1865                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
1866     return Error::success();                                                   \
1867   }
1868 #include "PassRegistry.def"
1869
1870   for (auto &C : CGSCCPipelineParsingCallbacks)
1871     if (C(Name, CGPM, InnerPipeline))
1872       return Error::success();
1873   return make_error<StringError>(
1874       formatv("unknown cgscc pass '{0}'", Name).str(),
1875       inconvertibleErrorCode());
1876 }
1877
1878 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
1879                                      const PipelineElement &E,
1880                                      bool VerifyEachPass, bool DebugLogging) {
1881   auto &Name = E.Name;
1882   auto &InnerPipeline = E.InnerPipeline;
1883
1884   // First handle complex passes like the pass managers which carry pipelines.
1885   if (!InnerPipeline.empty()) {
1886     if (Name == "function") {
1887       FunctionPassManager NestedFPM(DebugLogging);
1888       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1889                                                VerifyEachPass, DebugLogging))
1890         return Err;
1891       // Add the nested pass manager with the appropriate adaptor.
1892       FPM.addPass(std::move(NestedFPM));
1893       return Error::success();
1894     }
1895     if (Name == "loop") {
1896       LoopPassManager LPM(DebugLogging);
1897       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
1898                                            DebugLogging))
1899         return Err;
1900       // Add the nested pass manager with the appropriate adaptor.
1901       FPM.addPass(
1902           createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging));
1903       return Error::success();
1904     }
1905     if (auto Count = parseRepeatPassName(Name)) {
1906       FunctionPassManager NestedFPM(DebugLogging);
1907       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1908                                                VerifyEachPass, DebugLogging))
1909         return Err;
1910       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
1911       return Error::success();
1912     }
1913
1914     for (auto &C : FunctionPipelineParsingCallbacks)
1915       if (C(Name, FPM, InnerPipeline))
1916         return Error::success();
1917
1918     // Normal passes can't have pipelines.
1919     return make_error<StringError>(
1920         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
1921         inconvertibleErrorCode());
1922   }
1923
1924 // Now expand the basic registered passes from the .inc file.
1925 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1926   if (Name == NAME) {                                                          \
1927     FPM.addPass(CREATE_PASS);                                                  \
1928     return Error::success();                                                   \
1929   }
1930 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1931   if (checkParametrizedPassName(Name, NAME)) {                                 \
1932     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
1933     if (!Params)                                                               \
1934       return Params.takeError();                                               \
1935     FPM.addPass(CREATE_PASS(Params.get()));                                    \
1936     return Error::success();                                                   \
1937   }
1938 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1939   if (Name == "require<" NAME ">") {                                           \
1940     FPM.addPass(                                                               \
1941         RequireAnalysisPass<                                                   \
1942             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
1943     return Error::success();                                                   \
1944   }                                                                            \
1945   if (Name == "invalidate<" NAME ">") {                                        \
1946     FPM.addPass(InvalidateAnalysisPass<                                        \
1947                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
1948     return Error::success();                                                   \
1949   }
1950 #include "PassRegistry.def"
1951
1952   for (auto &C : FunctionPipelineParsingCallbacks)
1953     if (C(Name, FPM, InnerPipeline))
1954       return Error::success();
1955   return make_error<StringError>(
1956       formatv("unknown function pass '{0}'", Name).str(),
1957       inconvertibleErrorCode());
1958 }
1959
1960 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
1961                                  bool VerifyEachPass, bool DebugLogging) {
1962   StringRef Name = E.Name;
1963   auto &InnerPipeline = E.InnerPipeline;
1964
1965   // First handle complex passes like the pass managers which carry pipelines.
1966   if (!InnerPipeline.empty()) {
1967     if (Name == "loop") {
1968       LoopPassManager NestedLPM(DebugLogging);
1969       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
1970                                            VerifyEachPass, DebugLogging))
1971         return Err;
1972       // Add the nested pass manager with the appropriate adaptor.
1973       LPM.addPass(std::move(NestedLPM));
1974       return Error::success();
1975     }
1976     if (auto Count = parseRepeatPassName(Name)) {
1977       LoopPassManager NestedLPM(DebugLogging);
1978       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
1979                                            VerifyEachPass, DebugLogging))
1980         return Err;
1981       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
1982       return Error::success();
1983     }
1984
1985     for (auto &C : LoopPipelineParsingCallbacks)
1986       if (C(Name, LPM, InnerPipeline))
1987         return Error::success();
1988
1989     // Normal passes can't have pipelines.
1990     return make_error<StringError>(
1991         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
1992         inconvertibleErrorCode());
1993   }
1994
1995 // Now expand the basic registered passes from the .inc file.
1996 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
1997   if (Name == NAME) {                                                          \
1998     LPM.addPass(CREATE_PASS);                                                  \
1999     return Error::success();                                                   \
2000   }
2001 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2002   if (Name == "require<" NAME ">") {                                           \
2003     LPM.addPass(RequireAnalysisPass<                                           \
2004                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2005                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2006                 LPMUpdater &>());                                              \
2007     return Error::success();                                                   \
2008   }                                                                            \
2009   if (Name == "invalidate<" NAME ">") {                                        \
2010     LPM.addPass(InvalidateAnalysisPass<                                        \
2011                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2012     return Error::success();                                                   \
2013   }
2014 #include "PassRegistry.def"
2015
2016   for (auto &C : LoopPipelineParsingCallbacks)
2017     if (C(Name, LPM, InnerPipeline))
2018       return Error::success();
2019   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2020                                  inconvertibleErrorCode());
2021 }
2022
2023 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2024 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2025   if (Name == NAME) {                                                          \
2026     AA.registerModuleAnalysis<                                                 \
2027         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2028     return true;                                                               \
2029   }
2030 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2031   if (Name == NAME) {                                                          \
2032     AA.registerFunctionAnalysis<                                               \
2033         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2034     return true;                                                               \
2035   }
2036 #include "PassRegistry.def"
2037
2038   for (auto &C : AAParsingCallbacks)
2039     if (C(Name, AA))
2040       return true;
2041   return false;
2042 }
2043
2044 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2045                                          ArrayRef<PipelineElement> Pipeline,
2046                                          bool VerifyEachPass,
2047                                          bool DebugLogging) {
2048   for (const auto &Element : Pipeline) {
2049     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2050       return Err;
2051     // FIXME: No verifier support for Loop passes!
2052   }
2053   return Error::success();
2054 }
2055
2056 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2057                                              ArrayRef<PipelineElement> Pipeline,
2058                                              bool VerifyEachPass,
2059                                              bool DebugLogging) {
2060   for (const auto &Element : Pipeline) {
2061     if (auto Err =
2062             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2063       return Err;
2064     if (VerifyEachPass)
2065       FPM.addPass(VerifierPass());
2066   }
2067   return Error::success();
2068 }
2069
2070 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2071                                           ArrayRef<PipelineElement> Pipeline,
2072                                           bool VerifyEachPass,
2073                                           bool DebugLogging) {
2074   for (const auto &Element : Pipeline) {
2075     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2076       return Err;
2077     // FIXME: No verifier support for CGSCC passes!
2078   }
2079   return Error::success();
2080 }
2081
2082 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2083                                        FunctionAnalysisManager &FAM,
2084                                        CGSCCAnalysisManager &CGAM,
2085                                        ModuleAnalysisManager &MAM) {
2086   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2087   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2088   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2089   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2090   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2091   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2092   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2093 }
2094
2095 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2096                                            ArrayRef<PipelineElement> Pipeline,
2097                                            bool VerifyEachPass,
2098                                            bool DebugLogging) {
2099   for (const auto &Element : Pipeline) {
2100     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2101       return Err;
2102     if (VerifyEachPass)
2103       MPM.addPass(VerifierPass());
2104   }
2105   return Error::success();
2106 }
2107
2108 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2109 // FIXME: Should this routine accept a TargetMachine or require the caller to
2110 // pre-populate the analysis managers with target-specific stuff?
2111 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2112                                      StringRef PipelineText,
2113                                      bool VerifyEachPass, bool DebugLogging) {
2114   auto Pipeline = parsePipelineText(PipelineText);
2115   if (!Pipeline || Pipeline->empty())
2116     return make_error<StringError>(
2117         formatv("invalid pipeline '{0}'", PipelineText).str(),
2118         inconvertibleErrorCode());
2119
2120   // If the first name isn't at the module layer, wrap the pipeline up
2121   // automatically.
2122   StringRef FirstName = Pipeline->front().Name;
2123
2124   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2125     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2126       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2127     } else if (isFunctionPassName(FirstName,
2128                                   FunctionPipelineParsingCallbacks)) {
2129       Pipeline = {{"function", std::move(*Pipeline)}};
2130     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2131       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2132     } else {
2133       for (auto &C : TopLevelPipelineParsingCallbacks)
2134         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2135           return Error::success();
2136
2137       // Unknown pass or pipeline name!
2138       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2139       return make_error<StringError>(
2140           formatv("unknown {0} name '{1}'",
2141                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2142               .str(),
2143           inconvertibleErrorCode());
2144     }
2145   }
2146
2147   if (auto Err =
2148           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2149     return Err;
2150   return Error::success();
2151 }
2152
2153 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2154 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2155                                      StringRef PipelineText,
2156                                      bool VerifyEachPass, bool DebugLogging) {
2157   auto Pipeline = parsePipelineText(PipelineText);
2158   if (!Pipeline || Pipeline->empty())
2159     return make_error<StringError>(
2160         formatv("invalid pipeline '{0}'", PipelineText).str(),
2161         inconvertibleErrorCode());
2162
2163   StringRef FirstName = Pipeline->front().Name;
2164   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2165     return make_error<StringError>(
2166         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2167                 PipelineText)
2168             .str(),
2169         inconvertibleErrorCode());
2170
2171   if (auto Err =
2172           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2173     return Err;
2174   return Error::success();
2175 }
2176
2177 // Primary pass pipeline description parsing routine for a \c
2178 // FunctionPassManager
2179 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2180                                      StringRef PipelineText,
2181                                      bool VerifyEachPass, bool DebugLogging) {
2182   auto Pipeline = parsePipelineText(PipelineText);
2183   if (!Pipeline || Pipeline->empty())
2184     return make_error<StringError>(
2185         formatv("invalid pipeline '{0}'", PipelineText).str(),
2186         inconvertibleErrorCode());
2187
2188   StringRef FirstName = Pipeline->front().Name;
2189   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2190     return make_error<StringError>(
2191         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2192                 PipelineText)
2193             .str(),
2194         inconvertibleErrorCode());
2195
2196   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2197                                            DebugLogging))
2198     return Err;
2199   return Error::success();
2200 }
2201
2202 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2203 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2204                                      StringRef PipelineText,
2205                                      bool VerifyEachPass, bool DebugLogging) {
2206   auto Pipeline = parsePipelineText(PipelineText);
2207   if (!Pipeline || Pipeline->empty())
2208     return make_error<StringError>(
2209         formatv("invalid pipeline '{0}'", PipelineText).str(),
2210         inconvertibleErrorCode());
2211
2212   if (auto Err =
2213           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2214     return Err;
2215
2216   return Error::success();
2217 }
2218
2219 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2220   // If the pipeline just consists of the word 'default' just replace the AA
2221   // manager with our default one.
2222   if (PipelineText == "default") {
2223     AA = buildDefaultAAPipeline();
2224     return Error::success();
2225   }
2226
2227   while (!PipelineText.empty()) {
2228     StringRef Name;
2229     std::tie(Name, PipelineText) = PipelineText.split(',');
2230     if (!parseAAPassName(AA, Name))
2231       return make_error<StringError>(
2232           formatv("unknown alias analysis name '{0}'", Name).str(),
2233           inconvertibleErrorCode());
2234   }
2235
2236   return Error::success();
2237 }