lib/Passes/PassBuilder.cpp

   1 //===- Parsing, selection, and construction of pass pipelines -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 ///
  10 /// This file provides the implementation of the PassBuilder based on our
  11 /// static pass registry as well as related functionality. It also provides
  12 /// helpers to aid in analyzing, debugging, and testing passes and pass
  13 /// pipelines.
  14 ///
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "llvm/Passes/PassBuilder.h"
  18 #include "llvm/ADT/StringSwitch.h"
  19 #include "llvm/Analysis/AliasAnalysis.h"
  20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
  21 #include "llvm/Analysis/AssumptionCache.h"
  22 #include "llvm/Analysis/BasicAliasAnalysis.h"
  23 #include "llvm/Analysis/BlockFrequencyInfo.h"
  24 #include "llvm/Analysis/BranchProbabilityInfo.h"
  25 #include "llvm/Analysis/CFGPrinter.h"
  26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
  27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
  28 #include "llvm/Analysis/CGSCCPassManager.h"
  29 #include "llvm/Analysis/CallGraph.h"
  30 #include "llvm/Analysis/DemandedBits.h"
  31 #include "llvm/Analysis/DependenceAnalysis.h"
  32 #include "llvm/Analysis/DominanceFrontier.h"
  33 #include "llvm/Analysis/GlobalsModRef.h"
  34 #include "llvm/Analysis/IVUsers.h"
  35 #include "llvm/Analysis/LazyCallGraph.h"
  36 #include "llvm/Analysis/LazyValueInfo.h"
  37 #include "llvm/Analysis/LoopAccessAnalysis.h"
  38 #include "llvm/Analysis/LoopInfo.h"
  39 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
  40 #include "llvm/Analysis/MemorySSA.h"
  41 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  42 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  43 #include "llvm/Analysis/PhiValues.h"
  44 #include "llvm/Analysis/PostDominators.h"
  45 #include "llvm/Analysis/ProfileSummaryInfo.h"
  46 #include "llvm/Analysis/RegionInfo.h"
  47 #include "llvm/Analysis/ScalarEvolution.h"
  48 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
  49 #include "llvm/Analysis/ScopedNoAliasAA.h"
  50 #include "llvm/Analysis/StackSafetyAnalysis.h"
  51 #include "llvm/Analysis/TargetLibraryInfo.h"
  52 #include "llvm/Analysis/TargetTransformInfo.h"
  53 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
  54 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
  55 #include "llvm/CodeGen/UnreachableBlockElim.h"
  56 #include "llvm/IR/Dominators.h"
  57 #include "llvm/IR/IRPrintingPasses.h"
  58 #include "llvm/IR/PassManager.h"
  59 #include "llvm/IR/SafepointIRVerifier.h"
  60 #include "llvm/IR/Verifier.h"
  61 #include "llvm/Support/Debug.h"
  62 #include "llvm/Support/FormatVariadic.h"
  63 #include "llvm/Support/Regex.h"
  64 #include "llvm/Target/TargetMachine.h"
  65 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
  66 #include "llvm/Transforms/IPO/AlwaysInliner.h"
  67 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
  68 #include "llvm/Transforms/IPO/Attributor.h"
  69 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
  70 #include "llvm/Transforms/IPO/ConstantMerge.h"
  71 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
  72 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
  73 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
  74 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
  75 #include "llvm/Transforms/IPO/FunctionAttrs.h"
  76 #include "llvm/Transforms/IPO/FunctionImport.h"
  77 #include "llvm/Transforms/IPO/GlobalDCE.h"
  78 #include "llvm/Transforms/IPO/GlobalOpt.h"
  79 #include "llvm/Transforms/IPO/GlobalSplit.h"
  80 #include "llvm/Transforms/IPO/HotColdSplitting.h"
  81 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
  82 #include "llvm/Transforms/IPO/Inliner.h"
  83 #include "llvm/Transforms/IPO/Internalize.h"
  84 #include "llvm/Transforms/IPO/LowerTypeTests.h"
  85 #include "llvm/Transforms/IPO/PartialInlining.h"
  86 #include "llvm/Transforms/IPO/SCCP.h"
  87 #include "llvm/Transforms/IPO/SampleProfile.h"
  88 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
  89 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
  90 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  91 #include "llvm/Transforms/InstCombine/InstCombine.h"
  92 #include "llvm/Transforms/Instrumentation.h"
  93 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
  94 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
  95 #include "llvm/Transforms/Instrumentation/CGProfile.h"
  96 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
  97 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
  98 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  99 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
 100 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 101 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 102 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 103 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
 104 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 105 #include "llvm/Transforms/Scalar/ADCE.h"
 106 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
 107 #include "llvm/Transforms/Scalar/BDCE.h"
 108 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
 109 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
 110 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
 111 #include "llvm/Transforms/Scalar/DCE.h"
 112 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
 113 #include "llvm/Transforms/Scalar/DivRemPairs.h"
 114 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 115 #include "llvm/Transforms/Scalar/Float2Int.h"
 116 #include "llvm/Transforms/Scalar/GVN.h"
 117 #include "llvm/Transforms/Scalar/GuardWidening.h"
 118 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
 119 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
 120 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
 121 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
 122 #include "llvm/Transforms/Scalar/JumpThreading.h"
 123 #include "llvm/Transforms/Scalar/LICM.h"
 124 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
 125 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
 126 #include "llvm/Transforms/Scalar/LoopDeletion.h"
 127 #include "llvm/Transforms/Scalar/LoopDistribute.h"
 128 #include "llvm/Transforms/Scalar/LoopFuse.h"
 129 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
 130 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
 131 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
 132 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 133 #include "llvm/Transforms/Scalar/LoopPredication.h"
 134 #include "llvm/Transforms/Scalar/LoopRotation.h"
 135 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
 136 #include "llvm/Transforms/Scalar/LoopSink.h"
 137 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
 138 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
 139 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
 140 #include "llvm/Transforms/Scalar/LowerAtomic.h"
 141 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 142 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
 143 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
 144 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
 145 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
 146 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
 147 #include "llvm/Transforms/Scalar/MergeICmps.h"
 148 #include "llvm/Transforms/Scalar/NaryReassociate.h"
 149 #include "llvm/Transforms/Scalar/NewGVN.h"
 150 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
 151 #include "llvm/Transforms/Scalar/Reassociate.h"
 152 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
 153 #include "llvm/Transforms/Scalar/SCCP.h"
 154 #include "llvm/Transforms/Scalar/SROA.h"
 155 #include "llvm/Transforms/Scalar/Scalarizer.h"
 156 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
 157 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
 158 #include "llvm/Transforms/Scalar/Sink.h"
 159 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
 160 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
 161 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
 162 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
 163 #include "llvm/Transforms/Utils/AddDiscriminators.h"
 164 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
 165 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 166 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 167 #include "llvm/Transforms/Utils/LCSSA.h"
 168 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 169 #include "llvm/Transforms/Utils/LoopSimplify.h"
 170 #include "llvm/Transforms/Utils/LowerInvoke.h"
 171 #include "llvm/Transforms/Utils/Mem2Reg.h"
 172 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 173 #include "llvm/Transforms/Utils/SymbolRewriter.h"
 174 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 175 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 176 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
 177
 178 using namespace llvm;
 179
 180 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
 181                                              cl::ReallyHidden, cl::init(4));
 182 static cl::opt<bool>
 183     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
 184                        cl::Hidden, cl::ZeroOrMore,
 185                        cl::desc("Run Partial inlinining pass"));
 186
 187 static cl::opt<bool>
 188     RunNewGVN("enable-npm-newgvn", cl::init(false),
 189               cl::Hidden, cl::ZeroOrMore,
 190               cl::desc("Run NewGVN instead of GVN"));
 191
 192 static cl::opt<bool> EnableGVNHoist(
 193     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
 194     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 195
 196 static cl::opt<bool> EnableGVNSink(
 197     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
 198     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 199
 200 static cl::opt<bool> EnableUnrollAndJam(
 201     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
 202     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
 203
 204 static cl::opt<bool> EnableSyntheticCounts(
 205     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
 206     cl::desc("Run synthetic function entry count generation "
 207              "pass"));
 208
 209 static Regex DefaultAliasRegex(
 210     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
 211
 212 // This option is used in simplifying testing SampleFDO optimizations for
 213 // profile loading.
 214 static cl::opt<bool>
 215     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
 216               cl::desc("Enable control height reduction optimization (CHR)"));
 217
 218 PipelineTuningOptions::PipelineTuningOptions() {
 219   LoopInterleaving = EnableLoopInterleaving;
 220   LoopVectorization = EnableLoopVectorization;
 221   SLPVectorization = RunSLPVectorization;
 222   LoopUnrolling = true;
 223   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
 224   LicmMssaOptCap = SetLicmMssaOptCap;
 225   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
 226 }
 227
 228 extern cl::opt<bool> EnableHotColdSplit;
 229 extern cl::opt<bool> EnableOrderFileInstrumentation;
 230
 231 extern cl::opt<bool> FlattenedProfileUsed;
 232
 233 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
 234   switch (Level) {
 235   case PassBuilder::O0:
 236   case PassBuilder::O1:
 237   case PassBuilder::O2:
 238   case PassBuilder::O3:
 239     return false;
 240
 241   case PassBuilder::Os:
 242   case PassBuilder::Oz:
 243     return true;
 244   }
 245   llvm_unreachable("Invalid optimization level!");
 246 }
 247
 248 namespace {
 249
 250 /// No-op module pass which does nothing.
 251 struct NoOpModulePass {
 252   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
 253     return PreservedAnalyses::all();
 254   }
 255   static StringRef name() { return "NoOpModulePass"; }
 256 };
 257
 258 /// No-op module analysis.
 259 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
 260   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
 261   static AnalysisKey Key;
 262
 263 public:
 264   struct Result {};
 265   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
 266   static StringRef name() { return "NoOpModuleAnalysis"; }
 267 };
 268
 269 /// No-op CGSCC pass which does nothing.
 270 struct NoOpCGSCCPass {
 271   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
 272                         LazyCallGraph &, CGSCCUpdateResult &UR) {
 273     return PreservedAnalyses::all();
 274   }
 275   static StringRef name() { return "NoOpCGSCCPass"; }
 276 };
 277
 278 /// No-op CGSCC analysis.
 279 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
 280   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
 281   static AnalysisKey Key;
 282
 283 public:
 284   struct Result {};
 285   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
 286     return Result();
 287   }
 288   static StringRef name() { return "NoOpCGSCCAnalysis"; }
 289 };
 290
 291 /// No-op function pass which does nothing.
 292 struct NoOpFunctionPass {
 293   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
 294     return PreservedAnalyses::all();
 295   }
 296   static StringRef name() { return "NoOpFunctionPass"; }
 297 };
 298
 299 /// No-op function analysis.
 300 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
 301   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
 302   static AnalysisKey Key;
 303
 304 public:
 305   struct Result {};
 306   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
 307   static StringRef name() { return "NoOpFunctionAnalysis"; }
 308 };
 309
 310 /// No-op loop pass which does nothing.
 311 struct NoOpLoopPass {
 312   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
 313                         LoopStandardAnalysisResults &, LPMUpdater &) {
 314     return PreservedAnalyses::all();
 315   }
 316   static StringRef name() { return "NoOpLoopPass"; }
 317 };
 318
 319 /// No-op loop analysis.
 320 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
 321   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
 322   static AnalysisKey Key;
 323
 324 public:
 325   struct Result {};
 326   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
 327     return Result();
 328   }
 329   static StringRef name() { return "NoOpLoopAnalysis"; }
 330 };
 331
 332 AnalysisKey NoOpModuleAnalysis::Key;
 333 AnalysisKey NoOpCGSCCAnalysis::Key;
 334 AnalysisKey NoOpFunctionAnalysis::Key;
 335 AnalysisKey NoOpLoopAnalysis::Key;
 336
 337 } // End anonymous namespace.
 338
 339 void PassBuilder::invokePeepholeEPCallbacks(
 340     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
 341   for (auto &C : PeepholeEPCallbacks)
 342     C(FPM, Level);
 343 }
 344
 345 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
 346 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
 347   MAM.registerPass([&] { return CREATE_PASS; });
 348 #include "PassRegistry.def"
 349
 350   for (auto &C : ModuleAnalysisRegistrationCallbacks)
 351     C(MAM);
 352 }
 353
 354 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
 355 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
 356   CGAM.registerPass([&] { return CREATE_PASS; });
 357 #include "PassRegistry.def"
 358
 359   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
 360     C(CGAM);
 361 }
 362
 363 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
 364 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
 365   FAM.registerPass([&] { return CREATE_PASS; });
 366 #include "PassRegistry.def"
 367
 368   for (auto &C : FunctionAnalysisRegistrationCallbacks)
 369     C(FAM);
 370 }
 371
 372 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
 373 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 374   LAM.registerPass([&] { return CREATE_PASS; });
 375 #include "PassRegistry.def"
 376
 377   for (auto &C : LoopAnalysisRegistrationCallbacks)
 378     C(LAM);
 379 }
 380
 381 FunctionPassManager
 382 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 383                                                  ThinLTOPhase Phase,
 384                                                  bool DebugLogging) {
 385   assert(Level != O0 && "Must request optimizations!");
 386   FunctionPassManager FPM(DebugLogging);
 387
 388   // Form SSA out of local memory accesses after breaking apart aggregates into
 389   // scalars.
 390   FPM.addPass(SROA());
 391
 392   // Catch trivial redundancies
 393   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
 394
 395   // Hoisting of scalars and load expressions.
 396   if (EnableGVNHoist)
 397     FPM.addPass(GVNHoistPass());
 398
 399   // Global value numbering based sinking.
 400   if (EnableGVNSink) {
 401     FPM.addPass(GVNSinkPass());
 402     FPM.addPass(SimplifyCFGPass());
 403   }
 404
 405   // Speculative execution if the target has divergent branches; otherwise nop.
 406   FPM.addPass(SpeculativeExecutionPass());
 407
 408   // Optimize based on known information about branches, and cleanup afterward.
 409   FPM.addPass(JumpThreadingPass());
 410   FPM.addPass(CorrelatedValuePropagationPass());
 411   FPM.addPass(SimplifyCFGPass());
 412   if (Level == O3)
 413     FPM.addPass(AggressiveInstCombinePass());
 414   FPM.addPass(InstCombinePass());
 415
 416   if (!isOptimizingForSize(Level))
 417     FPM.addPass(LibCallsShrinkWrapPass());
 418
 419   invokePeepholeEPCallbacks(FPM, Level);
 420
 421   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
 422   // using the size value profile. Don't perform this when optimizing for size.
 423   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
 424       !isOptimizingForSize(Level))
 425     FPM.addPass(PGOMemOPSizeOpt());
 426
 427   FPM.addPass(TailCallElimPass());
 428   FPM.addPass(SimplifyCFGPass());
 429
 430   // Form canonically associated expression trees, and simplify the trees using
 431   // basic mathematical properties. For example, this will form (nearly)
 432   // minimal multiplication trees.
 433   FPM.addPass(ReassociatePass());
 434
 435   // Add the primary loop simplification pipeline.
 436   // FIXME: Currently this is split into two loop pass pipelines because we run
 437   // some function passes in between them. These can and should be removed
 438   // and/or replaced by scheduling the loop pass equivalents in the correct
 439   // positions. But those equivalent passes aren't powerful enough yet.
 440   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
 441   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
 442   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
 443   // `LoopInstSimplify`.
 444   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
 445
 446   // Simplify the loop body. We do this initially to clean up after other loop
 447   // passes run, either when iterating on a loop or on inner loops with
 448   // implications on the outer loop.
 449   LPM1.addPass(LoopInstSimplifyPass());
 450   LPM1.addPass(LoopSimplifyCFGPass());
 451
 452   // Rotate Loop - disable header duplication at -Oz
 453   LPM1.addPass(LoopRotatePass(Level != Oz));
 454   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
 455   LPM1.addPass(SimpleLoopUnswitchPass());
 456   LPM2.addPass(IndVarSimplifyPass());
 457   LPM2.addPass(LoopIdiomRecognizePass());
 458
 459   for (auto &C : LateLoopOptimizationsEPCallbacks)
 460     C(LPM2, Level);
 461
 462   LPM2.addPass(LoopDeletionPass());
 463   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
 464   // because it changes IR to makes profile annotation in back compile
 465   // inaccurate.
 466   if ((Phase != ThinLTOPhase::PreLink || !PGOOpt ||
 467        PGOOpt->Action != PGOOptions::SampleUse) &&
 468       PTO.LoopUnrolling)
 469     LPM2.addPass(
 470         LoopFullUnrollPass(Level, false, PTO.ForgetAllSCEVInLoopUnroll));
 471
 472   for (auto &C : LoopOptimizerEndEPCallbacks)
 473     C(LPM2, Level);
 474
 475   // We provide the opt remark emitter pass for LICM to use. We only need to do
 476   // this once as it is immutable.
 477   FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 478   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging));
 479   FPM.addPass(SimplifyCFGPass());
 480   FPM.addPass(InstCombinePass());
 481   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging));
 482
 483   // Eliminate redundancies.
 484   if (Level != O1) {
 485     // These passes add substantial compile time so skip them at O1.
 486     FPM.addPass(MergedLoadStoreMotionPass());
 487     if (RunNewGVN)
 488       FPM.addPass(NewGVNPass());
 489     else
 490       FPM.addPass(GVN());
 491   }
 492
 493   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
 494   FPM.addPass(MemCpyOptPass());
 495
 496   // Sparse conditional constant propagation.
 497   // FIXME: It isn't clear why we do this *after* loop passes rather than
 498   // before...
 499   FPM.addPass(SCCPPass());
 500
 501   // Delete dead bit computations (instcombine runs after to fold away the dead
 502   // computations, and then ADCE will run later to exploit any new DCE
 503   // opportunities that creates).
 504   FPM.addPass(BDCEPass());
 505
 506   // Run instcombine after redundancy and dead bit elimination to exploit
 507   // opportunities opened up by them.
 508   FPM.addPass(InstCombinePass());
 509   invokePeepholeEPCallbacks(FPM, Level);
 510
 511   // Re-consider control flow based optimizations after redundancy elimination,
 512   // redo DCE, etc.
 513   FPM.addPass(JumpThreadingPass());
 514   FPM.addPass(CorrelatedValuePropagationPass());
 515   FPM.addPass(DSEPass());
 516   FPM.addPass(createFunctionToLoopPassAdaptor(
 517       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
 518       DebugLogging));
 519
 520   for (auto &C : ScalarOptimizerLateEPCallbacks)
 521     C(FPM, Level);
 522
 523   // Finally, do an expensive DCE pass to catch all the dead code exposed by
 524   // the simplifications and basic cleanup after all the simplifications.
 525   FPM.addPass(ADCEPass());
 526   FPM.addPass(SimplifyCFGPass());
 527   FPM.addPass(InstCombinePass());
 528   invokePeepholeEPCallbacks(FPM, Level);
 529
 530   if (EnableCHR && Level == O3 && PGOOpt &&
 531       (PGOOpt->Action == PGOOptions::IRUse ||
 532        PGOOpt->Action == PGOOptions::SampleUse))
 533     FPM.addPass(ControlHeightReductionPass());
 534
 535   return FPM;
 536 }
 537
 538 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
 539                                     PassBuilder::OptimizationLevel Level,
 540                                     bool RunProfileGen, bool IsCS,
 541                                     std::string ProfileFile,
 542                                     std::string ProfileRemappingFile) {
 543   // Generally running simplification passes and the inliner with an high
 544   // threshold results in smaller executables, but there may be cases where
 545   // the size grows, so let's be conservative here and skip this simplification
 546   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
 547   // IsCS is true).
 548   if (!isOptimizingForSize(Level) && !IsCS) {
 549     InlineParams IP;
 550
 551     // In the old pass manager, this is a cl::opt. Should still this be one?
 552     IP.DefaultThreshold = 75;
 553
 554     // FIXME: The hint threshold has the same value used by the regular inliner.
 555     // This should probably be lowered after performance testing.
 556     // FIXME: this comment is cargo culted from the old pass manager, revisit).
 557     IP.HintThreshold = 325;
 558
 559     CGSCCPassManager CGPipeline(DebugLogging);
 560
 561     CGPipeline.addPass(InlinerPass(IP));
 562
 563     FunctionPassManager FPM;
 564     FPM.addPass(SROA());
 565     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
 566     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
 567     FPM.addPass(InstCombinePass()); // Combine silly sequences.
 568     invokePeepholeEPCallbacks(FPM, Level);
 569
 570     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
 571
 572     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
 573   }
 574
 575   // Delete anything that is now dead to make sure that we don't instrument
 576   // dead code. Instrumentation can end up keeping dead code around and
 577   // dramatically increase code size.
 578   MPM.addPass(GlobalDCEPass());
 579
 580   if (RunProfileGen) {
 581     MPM.addPass(PGOInstrumentationGen(IsCS));
 582
 583     FunctionPassManager FPM;
 584     FPM.addPass(
 585         createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 586     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 587
 588     // Add the profile lowering pass.
 589     InstrProfOptions Options;
 590     if (!ProfileFile.empty())
 591       Options.InstrProfileOutput = ProfileFile;
 592     Options.DoCounterPromotion = true;
 593     Options.UseBFIInPromotion = IsCS;
 594     MPM.addPass(InstrProfiling(Options, IsCS));
 595   } else if (!ProfileFile.empty()) {
 596     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
 597     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 598     // RequireAnalysisPass for PSI before subsequent non-module passes.
 599     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 600   }
 601 }
 602
 603 static InlineParams
 604 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
 605   auto O3 = PassBuilder::O3;
 606   unsigned OptLevel = Level > O3 ? 2 : Level;
 607   unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
 608   return getInlineParams(OptLevel, SizeLevel);
 609 }
 610
 611 ModulePassManager
 612 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
 613                                                ThinLTOPhase Phase,
 614                                                bool DebugLogging) {
 615   ModulePassManager MPM(DebugLogging);
 616
 617   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
 618
 619   // In ThinLTO mode, when flattened profile is used, all the available
 620   // profile information will be annotated in PreLink phase so there is
 621   // no need to load the profile again in PostLink.
 622   bool LoadSampleProfile =
 623       HasSampleProfile &&
 624       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
 625
 626   // During the ThinLTO backend phase we perform early indirect call promotion
 627   // here, before globalopt. Otherwise imported available_externally functions
 628   // look unreferenced and are removed. If we are going to load the sample
 629   // profile then defer until later.
 630   // TODO: See if we can move later and consolidate with the location where
 631   // we perform ICP when we are loading a sample profile.
 632   // TODO: We pass HasSampleProfile (whether there was a sample profile file
 633   // passed to the compile) to the SamplePGO flag of ICP. This is used to
 634   // determine whether the new direct calls are annotated with prof metadata.
 635   // Ideally this should be determined from whether the IR is annotated with
 636   // sample profile, and not whether the a sample profile was provided on the
 637   // command line. E.g. for flattened profiles where we will not be reloading
 638   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
 639   // provide the sample profile file.
 640   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
 641     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
 642
 643   // Do basic inference of function attributes from known properties of system
 644   // libraries and other oracles.
 645   MPM.addPass(InferFunctionAttrsPass());
 646
 647   // Create an early function pass manager to cleanup the output of the
 648   // frontend.
 649   FunctionPassManager EarlyFPM(DebugLogging);
 650   EarlyFPM.addPass(SimplifyCFGPass());
 651   EarlyFPM.addPass(SROA());
 652   EarlyFPM.addPass(EarlyCSEPass());
 653   EarlyFPM.addPass(LowerExpectIntrinsicPass());
 654   if (Level == O3)
 655     EarlyFPM.addPass(CallSiteSplittingPass());
 656
 657   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
 658   // to convert bitcast to direct calls so that they can be inlined during the
 659   // profile annotation prepration step.
 660   // More details about SamplePGO design can be found in:
 661   // https://research.google.com/pubs/pub45290.html
 662   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
 663   if (LoadSampleProfile)
 664     EarlyFPM.addPass(InstCombinePass());
 665   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
 666
 667   if (LoadSampleProfile) {
 668     // Annotate sample profile right after early FPM to ensure freshness of
 669     // the debug info.
 670     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
 671                                         PGOOpt->ProfileRemappingFile,
 672                                         Phase == ThinLTOPhase::PreLink));
 673     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 674     // RequireAnalysisPass for PSI before subsequent non-module passes.
 675     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 676     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
 677     // for the profile annotation to be accurate in the ThinLTO backend.
 678     if (Phase != ThinLTOPhase::PreLink)
 679       // We perform early indirect call promotion here, before globalopt.
 680       // This is important for the ThinLTO backend phase because otherwise
 681       // imported available_externally functions look unreferenced and are
 682       // removed.
 683       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
 684                                            true /* SamplePGO */));
 685   }
 686
 687   // Interprocedural constant propagation now that basic cleanup has occurred
 688   // and prior to optimizing globals.
 689   // FIXME: This position in the pipeline hasn't been carefully considered in
 690   // years, it should be re-analyzed.
 691   MPM.addPass(IPSCCPPass());
 692
 693   // Attach metadata to indirect call sites indicating the set of functions
 694   // they may target at run-time. This should follow IPSCCP.
 695   MPM.addPass(CalledValuePropagationPass());
 696
 697   // Optimize globals to try and fold them into constants.
 698   MPM.addPass(GlobalOptPass());
 699
 700   // Promote any localized globals to SSA registers.
 701   // FIXME: Should this instead by a run of SROA?
 702   // FIXME: We should probably run instcombine and simplify-cfg afterward to
 703   // delete control flows that are dead once globals have been folded to
 704   // constants.
 705   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
 706
 707   // Remove any dead arguments exposed by cleanups and constand folding
 708   // globals.
 709   MPM.addPass(DeadArgumentEliminationPass());
 710
 711   // Create a small function pass pipeline to cleanup after all the global
 712   // optimizations.
 713   FunctionPassManager GlobalCleanupPM(DebugLogging);
 714   GlobalCleanupPM.addPass(InstCombinePass());
 715   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
 716
 717   GlobalCleanupPM.addPass(SimplifyCFGPass());
 718   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
 719
 720   // Add all the requested passes for instrumentation PGO, if requested.
 721   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 722       (PGOOpt->Action == PGOOptions::IRInstr ||
 723        PGOOpt->Action == PGOOptions::IRUse)) {
 724     addPGOInstrPasses(MPM, DebugLogging, Level,
 725                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
 726                       /* IsCS */ false, PGOOpt->ProfileFile,
 727                       PGOOpt->ProfileRemappingFile);
 728     MPM.addPass(PGOIndirectCallPromotion(false, false));
 729   }
 730   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 731       PGOOpt->CSAction == PGOOptions::CSIRInstr)
 732     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
 733
 734   // Synthesize function entry counts for non-PGO compilation.
 735   if (EnableSyntheticCounts && !PGOOpt)
 736     MPM.addPass(SyntheticCountsPropagation());
 737
 738   // Require the GlobalsAA analysis for the module so we can query it within
 739   // the CGSCC pipeline.
 740   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 741
 742   // Require the ProfileSummaryAnalysis for the module so we can query it within
 743   // the inliner pass.
 744   MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 745
 746   // Now begin the main postorder CGSCC pipeline.
 747   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
 748   // manager and trying to emulate its precise behavior. Much of this doesn't
 749   // make a lot of sense and we should revisit the core CGSCC structure.
 750   CGSCCPassManager MainCGPipeline(DebugLogging);
 751
 752   // Note: historically, the PruneEH pass was run first to deduce nounwind and
 753   // generally clean up exception handling overhead. It isn't clear this is
 754   // valuable as the inliner doesn't currently care whether it is inlining an
 755   // invoke or a call.
 756
 757   // Run the inliner first. The theory is that we are walking bottom-up and so
 758   // the callees have already been fully optimized, and we want to inline them
 759   // into the callers so that our optimizations can reflect that.
 760   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
 761   // because it makes profile annotation in the backend inaccurate.
 762   InlineParams IP = getInlineParamsFromOptLevel(Level);
 763   if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
 764       PGOOpt->Action == PGOOptions::SampleUse)
 765     IP.HotCallSiteThreshold = 0;
 766   MainCGPipeline.addPass(InlinerPass(IP));
 767
 768   // Now deduce any function attributes based in the current code.
 769   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
 770
 771   // When at O3 add argument promotion to the pass pipeline.
 772   // FIXME: It isn't at all clear why this should be limited to O3.
 773   if (Level == O3)
 774     MainCGPipeline.addPass(ArgumentPromotionPass());
 775
 776   // Lastly, add the core function simplification pipeline nested inside the
 777   // CGSCC walk.
 778   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
 779       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
 780
 781   for (auto &C : CGSCCOptimizerLateEPCallbacks)
 782     C(MainCGPipeline, Level);
 783
 784   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
 785   // to detect when we devirtualize indirect calls and iterate the SCC passes
 786   // in that case to try and catch knock-on inlining or function attrs
 787   // opportunities. Then we add it to the module pipeline by walking the SCCs
 788   // in postorder (or bottom-up).
 789   MPM.addPass(
 790       createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
 791           std::move(MainCGPipeline), MaxDevirtIterations)));
 792
 793   return MPM;
 794 }
 795
 796 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
 797     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
 798   ModulePassManager MPM(DebugLogging);
 799
 800   // Optimize globals now that the module is fully simplified.
 801   MPM.addPass(GlobalOptPass());
 802   MPM.addPass(GlobalDCEPass());
 803
 804   // Run partial inlining pass to partially inline functions that have
 805   // large bodies.
 806   if (RunPartialInlining)
 807     MPM.addPass(PartialInlinerPass());
 808
 809   // Remove avail extern fns and globals definitions since we aren't compiling
 810   // an object file for later LTO. For LTO we want to preserve these so they
 811   // are eligible for inlining at link-time. Note if they are unreferenced they
 812   // will be removed by GlobalDCE later, so this only impacts referenced
 813   // available externally globals. Eventually they will be suppressed during
 814   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
 815   // may make globals referenced by available external functions dead and saves
 816   // running remaining passes on the eliminated functions. These should be
 817   // preserved during prelinking for link-time inlining decisions.
 818   if (!LTOPreLink)
 819     MPM.addPass(EliminateAvailableExternallyPass());
 820
 821   if (EnableOrderFileInstrumentation)
 822     MPM.addPass(InstrOrderFilePass());
 823
 824   // Do RPO function attribute inference across the module to forward-propagate
 825   // attributes where applicable.
 826   // FIXME: Is this really an optimization rather than a canonicalization?
 827   MPM.addPass(ReversePostOrderFunctionAttrsPass());
 828
 829   // Do a post inline PGO instrumentation and use pass. This is a context
 830   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
 831   // cross-module inline has not been done yet. The context sensitive
 832   // instrumentation is after all the inlines are done.
 833   if (!LTOPreLink && PGOOpt) {
 834     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
 835       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
 836                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
 837                         PGOOpt->ProfileRemappingFile);
 838     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
 839       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
 840                         /* IsCS */ true, PGOOpt->ProfileFile,
 841                         PGOOpt->ProfileRemappingFile);
 842   }
 843
 844   // Re-require GloblasAA here prior to function passes. This is particularly
 845   // useful as the above will have inlined, DCE'ed, and function-attr
 846   // propagated everything. We should at this point have a reasonably minimal
 847   // and richly annotated call graph. By computing aliasing and mod/ref
 848   // information for all local globals here, the late loop passes and notably
 849   // the vectorizer will be able to use them to help recognize vectorizable
 850   // memory operations.
 851   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 852
 853   FunctionPassManager OptimizePM(DebugLogging);
 854   OptimizePM.addPass(Float2IntPass());
 855   // FIXME: We need to run some loop optimizations to re-rotate loops after
 856   // simplify-cfg and others undo their rotation.
 857
 858   // Optimize the loop execution. These passes operate on entire loop nests
 859   // rather than on each loop in an inside-out manner, and so they are actually
 860   // function passes.
 861
 862   for (auto &C : VectorizerStartEPCallbacks)
 863     C(OptimizePM, Level);
 864
 865   // First rotate loops that may have been un-rotated by prior passes.
 866   OptimizePM.addPass(
 867       createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 868
 869   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
 870   // into separate loop that would otherwise inhibit vectorization.  This is
 871   // currently only performed for loops marked with the metadata
 872   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
 873   OptimizePM.addPass(LoopDistributePass());
 874
 875   // Now run the core loop vectorizer.
 876   OptimizePM.addPass(LoopVectorizePass(
 877       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
 878
 879   // Eliminate loads by forwarding stores from the previous iteration to loads
 880   // of the current iteration.
 881   OptimizePM.addPass(LoopLoadEliminationPass());
 882
 883   // Cleanup after the loop optimization passes.
 884   OptimizePM.addPass(InstCombinePass());
 885
 886   // Now that we've formed fast to execute loop structures, we do further
 887   // optimizations. These are run afterward as they might block doing complex
 888   // analyses and transforms such as what are needed for loop vectorization.
 889
 890   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
 891   // GVN, loop transforms, and others have already run, so it's now better to
 892   // convert to more optimized IR using more aggressive simplify CFG options.
 893   // The extra sinking transform can create larger basic blocks, so do this
 894   // before SLP vectorization.
 895   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
 896                                      forwardSwitchCondToPhi(true).
 897                                      convertSwitchToLookupTable(true).
 898                                      needCanonicalLoops(false).
 899                                      sinkCommonInsts(true)));
 900
 901   // Optimize parallel scalar instruction chains into SIMD instructions.
 902   if (PTO.SLPVectorization)
 903     OptimizePM.addPass(SLPVectorizerPass());
 904
 905   OptimizePM.addPass(InstCombinePass());
 906
 907   // Unroll small loops to hide loop backedge latency and saturate any parallel
 908   // execution resources of an out-of-order processor. We also then need to
 909   // clean up redundancies and loop invariant code.
 910   // FIXME: It would be really good to use a loop-integrated instruction
 911   // combiner for cleanup here so that the unrolling and LICM can be pipelined
 912   // across the loop nests.
 913   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
 914   if (EnableUnrollAndJam) {
 915     OptimizePM.addPass(
 916         createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level)));
 917   }
 918   if (PTO.LoopUnrolling)
 919     OptimizePM.addPass(LoopUnrollPass(
 920         LoopUnrollOptions(Level, false, PTO.ForgetAllSCEVInLoopUnroll)));
 921   OptimizePM.addPass(WarnMissedTransformationsPass());
 922   OptimizePM.addPass(InstCombinePass());
 923   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 924   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
 925       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
 926       DebugLogging));
 927
 928   // Now that we've vectorized and unrolled loops, we may have more refined
 929   // alignment information, try to re-derive it here.
 930   OptimizePM.addPass(AlignmentFromAssumptionsPass());
 931
 932   // Split out cold code. Splitting is done late to avoid hiding context from
 933   // other optimizations and inadvertently regressing performance. The tradeoff
 934   // is that this has a higher code size cost than splitting early.
 935   if (EnableHotColdSplit && !LTOPreLink)
 936     MPM.addPass(HotColdSplittingPass());
 937
 938   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
 939   // canonicalization pass that enables other optimizations. As a result,
 940   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
 941   // result too early.
 942   OptimizePM.addPass(LoopSinkPass());
 943
 944   // And finally clean up LCSSA form before generating code.
 945   OptimizePM.addPass(InstSimplifyPass());
 946
 947   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
 948   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
 949   // flattening of blocks.
 950   OptimizePM.addPass(DivRemPairsPass());
 951
 952   // LoopSink (and other loop passes since the last simplifyCFG) might have
 953   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
 954   OptimizePM.addPass(SimplifyCFGPass());
 955
 956   // Optimize PHIs by speculating around them when profitable. Note that this
 957   // pass needs to be run after any PRE or similar pass as it is essentially
 958   // inserting redundancies into the program. This even includes SimplifyCFG.
 959   OptimizePM.addPass(SpeculateAroundPHIsPass());
 960
 961   for (auto &C : OptimizerLastEPCallbacks)
 962     C(OptimizePM, Level);
 963
 964   // Add the core optimizing pipeline.
 965   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
 966
 967   MPM.addPass(CGProfilePass());
 968
 969   // Now we need to do some global optimization transforms.
 970   // FIXME: It would seem like these should come first in the optimization
 971   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
 972   // ordering here.
 973   MPM.addPass(GlobalDCEPass());
 974   MPM.addPass(ConstantMergePass());
 975
 976   return MPM;
 977 }
 978
 979 ModulePassManager
 980 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
 981                                            bool DebugLogging, bool LTOPreLink) {
 982   assert(Level != O0 && "Must request optimizations for the default pipeline!");
 983
 984   ModulePassManager MPM(DebugLogging);
 985
 986   // Force any function attributes we want the rest of the pipeline to observe.
 987   MPM.addPass(ForceFunctionAttrsPass());
 988
 989   // Apply module pipeline start EP callback.
 990   for (auto &C : PipelineStartEPCallbacks)
 991     C(MPM);
 992
 993   if (PGOOpt && PGOOpt->SamplePGOSupport)
 994     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 995
 996   // Add the core simplification pipeline.
 997   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
 998                                                 DebugLogging));
 999
1000   // Now add the optimization pipeline.
1001   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1002
1003   return MPM;
1004 }
1005
1006 ModulePassManager
1007 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1008                                                 bool DebugLogging) {
1009   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1010
1011   ModulePassManager MPM(DebugLogging);
1012
1013   // Force any function attributes we want the rest of the pipeline to observe.
1014   MPM.addPass(ForceFunctionAttrsPass());
1015
1016   if (PGOOpt && PGOOpt->SamplePGOSupport)
1017     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1018
1019   // Apply module pipeline start EP callback.
1020   for (auto &C : PipelineStartEPCallbacks)
1021     C(MPM);
1022
1023   // If we are planning to perform ThinLTO later, we don't bloat the code with
1024   // unrolling/vectorization/... now. Just simplify the module as much as we
1025   // can.
1026   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1027                                                 DebugLogging));
1028
1029   // Run partial inlining pass to partially inline functions that have
1030   // large bodies.
1031   // FIXME: It isn't clear whether this is really the right place to run this
1032   // in ThinLTO. Because there is another canonicalization and simplification
1033   // phase that will run after the thin link, running this here ends up with
1034   // less information than will be available later and it may grow functions in
1035   // ways that aren't beneficial.
1036   if (RunPartialInlining)
1037     MPM.addPass(PartialInlinerPass());
1038
1039   // Reduce the size of the IR as much as possible.
1040   MPM.addPass(GlobalOptPass());
1041
1042   return MPM;
1043 }
1044
1045 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1046     OptimizationLevel Level, bool DebugLogging,
1047     const ModuleSummaryIndex *ImportSummary) {
1048   ModulePassManager MPM(DebugLogging);
1049
1050   if (ImportSummary) {
1051     // These passes import type identifier resolutions for whole-program
1052     // devirtualization and CFI. They must run early because other passes may
1053     // disturb the specific instruction patterns that these passes look for,
1054     // creating dependencies on resolutions that may not appear in the summary.
1055     //
1056     // For example, GVN may transform the pattern assume(type.test) appearing in
1057     // two basic blocks into assume(phi(type.test, type.test)), which would
1058     // transform a dependency on a WPD resolution into a dependency on a type
1059     // identifier resolution for CFI.
1060     //
1061     // Also, WPD has access to more precise information than ICP and can
1062     // devirtualize more effectively, so it should operate on the IR first.
1063     //
1064     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1065     // metadata and intrinsics.
1066     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1067     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1068   }
1069
1070   if (Level == O0)
1071     return MPM;
1072
1073   // Force any function attributes we want the rest of the pipeline to observe.
1074   MPM.addPass(ForceFunctionAttrsPass());
1075
1076   // Add the core simplification pipeline.
1077   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1078                                                 DebugLogging));
1079
1080   // Now add the optimization pipeline.
1081   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1082
1083   return MPM;
1084 }
1085
1086 ModulePassManager
1087 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1088                                             bool DebugLogging) {
1089   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1090   // FIXME: We should use a customized pre-link pipeline!
1091   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1092                                        /* LTOPreLink */true);
1093 }
1094
1095 ModulePassManager
1096 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1097                                      ModuleSummaryIndex *ExportSummary) {
1098   ModulePassManager MPM(DebugLogging);
1099
1100   if (Level == O0) {
1101     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1102     // metadata and intrinsics.
1103     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1104     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1105     return MPM;
1106   }
1107
1108   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1109     // Load sample profile before running the LTO optimization pipeline.
1110     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1111                                         PGOOpt->ProfileRemappingFile,
1112                                         false /* ThinLTOPhase::PreLink */));
1113     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1114     // RequireAnalysisPass for PSI before subsequent non-module passes.
1115     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1116   }
1117
1118   // Remove unused virtual tables to improve the quality of code generated by
1119   // whole-program devirtualization and bitset lowering.
1120   MPM.addPass(GlobalDCEPass());
1121
1122   // Force any function attributes we want the rest of the pipeline to observe.
1123   MPM.addPass(ForceFunctionAttrsPass());
1124
1125   // Do basic inference of function attributes from known properties of system
1126   // libraries and other oracles.
1127   MPM.addPass(InferFunctionAttrsPass());
1128
1129   if (Level > 1) {
1130     FunctionPassManager EarlyFPM(DebugLogging);
1131     EarlyFPM.addPass(CallSiteSplittingPass());
1132     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1133
1134     // Indirect call promotion. This should promote all the targets that are
1135     // left by the earlier promotion pass that promotes intra-module targets.
1136     // This two-step promotion is to save the compile time. For LTO, it should
1137     // produce the same result as if we only do promotion here.
1138     MPM.addPass(PGOIndirectCallPromotion(
1139         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1140     // Propagate constants at call sites into the functions they call.  This
1141     // opens opportunities for globalopt (and inlining) by substituting function
1142     // pointers passed as arguments to direct uses of functions.
1143    MPM.addPass(IPSCCPPass());
1144
1145    // Attach metadata to indirect call sites indicating the set of functions
1146    // they may target at run-time. This should follow IPSCCP.
1147    MPM.addPass(CalledValuePropagationPass());
1148   }
1149
1150   // Now deduce any function attributes based in the current code.
1151   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1152               PostOrderFunctionAttrsPass()));
1153
1154   // Do RPO function attribute inference across the module to forward-propagate
1155   // attributes where applicable.
1156   // FIXME: Is this really an optimization rather than a canonicalization?
1157   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1158
1159   // Use in-range annotations on GEP indices to split globals where beneficial.
1160   MPM.addPass(GlobalSplitPass());
1161
1162   // Run whole program optimization of virtual call when the list of callees
1163   // is fixed.
1164   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1165
1166   // Stop here at -O1.
1167   if (Level == 1) {
1168     // The LowerTypeTestsPass needs to run to lower type metadata and the
1169     // type.test intrinsics. The pass does nothing if CFI is disabled.
1170     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1171     return MPM;
1172   }
1173
1174   // Optimize globals to try and fold them into constants.
1175   MPM.addPass(GlobalOptPass());
1176
1177   // Promote any localized globals to SSA registers.
1178   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1179
1180   // Linking modules together can lead to duplicate global constant, only
1181   // keep one copy of each constant.
1182   MPM.addPass(ConstantMergePass());
1183
1184   // Remove unused arguments from functions.
1185   MPM.addPass(DeadArgumentEliminationPass());
1186
1187   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1188   // simplification opportunities, and both can propagate functions through
1189   // function pointers.  When this happens, we often have to resolve varargs
1190   // calls, etc, so let instcombine do this.
1191   FunctionPassManager PeepholeFPM(DebugLogging);
1192   if (Level == O3)
1193     PeepholeFPM.addPass(AggressiveInstCombinePass());
1194   PeepholeFPM.addPass(InstCombinePass());
1195   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1196
1197   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1198
1199   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1200   // generally clean up exception handling overhead. It isn't clear this is
1201   // valuable as the inliner doesn't currently care whether it is inlining an
1202   // invoke or a call.
1203   // Run the inliner now.
1204   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1205       InlinerPass(getInlineParamsFromOptLevel(Level))));
1206
1207   // Optimize globals again after we ran the inliner.
1208   MPM.addPass(GlobalOptPass());
1209
1210   // Garbage collect dead functions.
1211   // FIXME: Add ArgumentPromotion pass after once it's ported.
1212   MPM.addPass(GlobalDCEPass());
1213
1214   FunctionPassManager FPM(DebugLogging);
1215   // The IPO Passes may leave cruft around. Clean up after them.
1216   FPM.addPass(InstCombinePass());
1217   invokePeepholeEPCallbacks(FPM, Level);
1218
1219   FPM.addPass(JumpThreadingPass());
1220
1221   // Do a post inline PGO instrumentation and use pass. This is a context
1222   // sensitive PGO pass.
1223   if (PGOOpt) {
1224     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1225       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1226                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1227                         PGOOpt->ProfileRemappingFile);
1228     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1229       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1230                         /* IsCS */ true, PGOOpt->ProfileFile,
1231                         PGOOpt->ProfileRemappingFile);
1232   }
1233
1234   // Break up allocas
1235   FPM.addPass(SROA());
1236
1237   // LTO provides additional opportunities for tailcall elimination due to
1238   // link-time inlining, and visibility of nocapture attribute.
1239   FPM.addPass(TailCallElimPass());
1240
1241   // Run a few AA driver optimizations here and now to cleanup the code.
1242   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1243
1244   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1245               PostOrderFunctionAttrsPass()));
1246   // FIXME: here we run IP alias analysis in the legacy PM.
1247
1248   FunctionPassManager MainFPM;
1249
1250   // FIXME: once we fix LoopPass Manager, add LICM here.
1251   // FIXME: once we provide support for enabling MLSM, add it here.
1252   if (RunNewGVN)
1253     MainFPM.addPass(NewGVNPass());
1254   else
1255     MainFPM.addPass(GVN());
1256
1257   // Remove dead memcpy()'s.
1258   MainFPM.addPass(MemCpyOptPass());
1259
1260   // Nuke dead stores.
1261   MainFPM.addPass(DSEPass());
1262
1263   // FIXME: at this point, we run a bunch of loop passes:
1264   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1265   // loopVectorize. Enable them once the remaining issue with LPM
1266   // are sorted out.
1267
1268   MainFPM.addPass(InstCombinePass());
1269   MainFPM.addPass(SimplifyCFGPass());
1270   MainFPM.addPass(SCCPPass());
1271   MainFPM.addPass(InstCombinePass());
1272   MainFPM.addPass(BDCEPass());
1273
1274   // FIXME: We may want to run SLPVectorizer here.
1275   // After vectorization, assume intrinsics may tell us more
1276   // about pointer alignments.
1277 #if 0
1278   MainFPM.add(AlignmentFromAssumptionsPass());
1279 #endif
1280
1281   // FIXME: Conditionally run LoadCombine here, after it's ported
1282   // (in case we still have this pass, given its questionable usefulness).
1283
1284   MainFPM.addPass(InstCombinePass());
1285   invokePeepholeEPCallbacks(MainFPM, Level);
1286   MainFPM.addPass(JumpThreadingPass());
1287   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1288
1289   // Create a function that performs CFI checks for cross-DSO calls with
1290   // targets in the current module.
1291   MPM.addPass(CrossDSOCFIPass());
1292
1293   // Lower type metadata and the type.test intrinsic. This pass supports
1294   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1295   // to be run at link time if CFI is enabled. This pass does nothing if
1296   // CFI is disabled.
1297   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1298
1299   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1300   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1301   if (EnableHotColdSplit)
1302     MPM.addPass(HotColdSplittingPass());
1303
1304   // Add late LTO optimization passes.
1305   // Delete basic blocks, which optimization passes may have killed.
1306   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1307
1308   // Drop bodies of available eternally objects to improve GlobalDCE.
1309   MPM.addPass(EliminateAvailableExternallyPass());
1310
1311   // Now that we have optimized the program, discard unreachable functions.
1312   MPM.addPass(GlobalDCEPass());
1313
1314   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1315   return MPM;
1316 }
1317
1318 AAManager PassBuilder::buildDefaultAAPipeline() {
1319   AAManager AA;
1320
1321   // The order in which these are registered determines their priority when
1322   // being queried.
1323
1324   // First we register the basic alias analysis that provides the majority of
1325   // per-function local AA logic. This is a stateless, on-demand local set of
1326   // AA techniques.
1327   AA.registerFunctionAnalysis<BasicAA>();
1328
1329   // Next we query fast, specialized alias analyses that wrap IR-embedded
1330   // information about aliasing.
1331   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1332   AA.registerFunctionAnalysis<TypeBasedAA>();
1333
1334   // Add support for querying global aliasing information when available.
1335   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1336   // analysis, all that the `AAManager` can do is query for any *cached*
1337   // results from `GlobalsAA` through a readonly proxy.
1338   AA.registerModuleAnalysis<GlobalsAA>();
1339
1340   return AA;
1341 }
1342
1343 static Optional<int> parseRepeatPassName(StringRef Name) {
1344   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1345     return None;
1346   int Count;
1347   if (Name.getAsInteger(0, Count) || Count <= 0)
1348     return None;
1349   return Count;
1350 }
1351
1352 static Optional<int> parseDevirtPassName(StringRef Name) {
1353   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1354     return None;
1355   int Count;
1356   if (Name.getAsInteger(0, Count) || Count <= 0)
1357     return None;
1358   return Count;
1359 }
1360
1361 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1362   if (!Name.consume_front(PassName))
1363     return false;
1364   // normal pass name w/o parameters == default parameters
1365   if (Name.empty())
1366     return true;
1367   return Name.startswith("<") && Name.endswith(">");
1368 }
1369
1370 namespace {
1371
1372 /// This performs customized parsing of pass name with parameters.
1373 ///
1374 /// We do not need parametrization of passes in textual pipeline very often,
1375 /// yet on a rare occasion ability to specify parameters right there can be
1376 /// useful.
1377 ///
1378 /// \p Name - parameterized specification of a pass from a textual pipeline
1379 /// is a string in a form of :
1380 ///      PassName '<' parameter-list '>'
1381 ///
1382 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1383 /// It takes a string-ref of parameters and returns either StringError or a
1384 /// parameter list in a form of a custom parameters type, all wrapped into
1385 /// Expected<> template class.
1386 ///
1387 template <typename ParametersParseCallableT>
1388 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1389                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1390   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1391
1392   StringRef Params = Name;
1393   if (!Params.consume_front(PassName)) {
1394     assert(false &&
1395            "unable to strip pass name from parametrized pass specification");
1396   }
1397   if (Params.empty())
1398     return ParametersT{};
1399   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1400     assert(false && "invalid format for parametrized pass name");
1401   }
1402
1403   Expected<ParametersT> Result = Parser(Params);
1404   assert((Result || Result.template errorIsA<StringError>()) &&
1405          "Pass parameter parser can only return StringErrors.");
1406   return std::move(Result);
1407 }
1408
1409 /// Parser of parameters for LoopUnroll pass.
1410 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1411   LoopUnrollOptions UnrollOpts;
1412   while (!Params.empty()) {
1413     StringRef ParamName;
1414     std::tie(ParamName, Params) = Params.split(';');
1415     int OptLevel = StringSwitch<int>(ParamName)
1416                        .Case("O0", 0)
1417                        .Case("O1", 1)
1418                        .Case("O2", 2)
1419                        .Case("O3", 3)
1420                        .Default(-1);
1421     if (OptLevel >= 0) {
1422       UnrollOpts.setOptLevel(OptLevel);
1423       continue;
1424     }
1425
1426     bool Enable = !ParamName.consume_front("no-");
1427     if (ParamName == "partial") {
1428       UnrollOpts.setPartial(Enable);
1429     } else if (ParamName == "peeling") {
1430       UnrollOpts.setPeeling(Enable);
1431     } else if (ParamName == "runtime") {
1432       UnrollOpts.setRuntime(Enable);
1433     } else if (ParamName == "upperbound") {
1434       UnrollOpts.setUpperBound(Enable);
1435     } else {
1436       return make_error<StringError>(
1437           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1438           inconvertibleErrorCode());
1439     }
1440   }
1441   return UnrollOpts;
1442 }
1443
1444 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1445   MemorySanitizerOptions Result;
1446   while (!Params.empty()) {
1447     StringRef ParamName;
1448     std::tie(ParamName, Params) = Params.split(';');
1449
1450     if (ParamName == "recover") {
1451       Result.Recover = true;
1452     } else if (ParamName == "kernel") {
1453       Result.Kernel = true;
1454     } else if (ParamName.consume_front("track-origins=")) {
1455       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1456         return make_error<StringError>(
1457             formatv("invalid argument to MemorySanitizer pass track-origins "
1458                     "parameter: '{0}' ",
1459                     ParamName)
1460                 .str(),
1461             inconvertibleErrorCode());
1462     } else {
1463       return make_error<StringError>(
1464           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1465               .str(),
1466           inconvertibleErrorCode());
1467     }
1468   }
1469   return Result;
1470 }
1471
1472 /// Parser of parameters for SimplifyCFG pass.
1473 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1474   SimplifyCFGOptions Result;
1475   while (!Params.empty()) {
1476     StringRef ParamName;
1477     std::tie(ParamName, Params) = Params.split(';');
1478
1479     bool Enable = !ParamName.consume_front("no-");
1480     if (ParamName == "forward-switch-cond") {
1481       Result.forwardSwitchCondToPhi(Enable);
1482     } else if (ParamName == "switch-to-lookup") {
1483       Result.convertSwitchToLookupTable(Enable);
1484     } else if (ParamName == "keep-loops") {
1485       Result.needCanonicalLoops(Enable);
1486     } else if (ParamName == "sink-common-insts") {
1487       Result.sinkCommonInsts(Enable);
1488     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1489       APInt BonusInstThreshold;
1490       if (ParamName.getAsInteger(0, BonusInstThreshold))
1491         return make_error<StringError>(
1492             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1493                     "parameter: '{0}' ",
1494                     ParamName).str(),
1495             inconvertibleErrorCode());
1496       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1497     } else {
1498       return make_error<StringError>(
1499           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1500           inconvertibleErrorCode());
1501     }
1502   }
1503   return Result;
1504 }
1505
1506 /// Parser of parameters for LoopVectorize pass.
1507 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1508   LoopVectorizeOptions Opts;
1509   while (!Params.empty()) {
1510     StringRef ParamName;
1511     std::tie(ParamName, Params) = Params.split(';');
1512
1513     bool Enable = !ParamName.consume_front("no-");
1514     if (ParamName == "interleave-forced-only") {
1515       Opts.setInterleaveOnlyWhenForced(Enable);
1516     } else if (ParamName == "vectorize-forced-only") {
1517       Opts.setVectorizeOnlyWhenForced(Enable);
1518     } else {
1519       return make_error<StringError>(
1520           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1521           inconvertibleErrorCode());
1522     }
1523   }
1524   return Opts;
1525 }
1526
1527 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1528   bool Result = false;
1529   while (!Params.empty()) {
1530     StringRef ParamName;
1531     std::tie(ParamName, Params) = Params.split(';');
1532
1533     bool Enable = !ParamName.consume_front("no-");
1534     if (ParamName == "nontrivial") {
1535       Result = Enable;
1536     } else {
1537       return make_error<StringError>(
1538           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1539               .str(),
1540           inconvertibleErrorCode());
1541     }
1542   }
1543   return Result;
1544 }
1545 } // namespace
1546
1547 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1548 /// alias.
1549 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1550   return Name.startswith("default") || Name.startswith("thinlto") ||
1551          Name.startswith("lto");
1552 }
1553
1554 /// Tests whether registered callbacks will accept a given pass name.
1555 ///
1556 /// When parsing a pipeline text, the type of the outermost pipeline may be
1557 /// omitted, in which case the type is automatically determined from the first
1558 /// pass name in the text. This may be a name that is handled through one of the
1559 /// callbacks. We check this through the oridinary parsing callbacks by setting
1560 /// up a dummy PassManager in order to not force the client to also handle this
1561 /// type of query.
1562 template <typename PassManagerT, typename CallbacksT>
1563 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1564   if (!Callbacks.empty()) {
1565     PassManagerT DummyPM;
1566     for (auto &CB : Callbacks)
1567       if (CB(Name, DummyPM, {}))
1568         return true;
1569   }
1570   return false;
1571 }
1572
1573 template <typename CallbacksT>
1574 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1575   // Manually handle aliases for pre-configured pipeline fragments.
1576   if (startsWithDefaultPipelineAliasPrefix(Name))
1577     return DefaultAliasRegex.match(Name);
1578
1579   // Explicitly handle pass manager names.
1580   if (Name == "module")
1581     return true;
1582   if (Name == "cgscc")
1583     return true;
1584   if (Name == "function")
1585     return true;
1586
1587   // Explicitly handle custom-parsed pass names.
1588   if (parseRepeatPassName(Name))
1589     return true;
1590
1591 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1592   if (Name == NAME)                                                            \
1593     return true;
1594 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1595   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1596     return true;
1597 #include "PassRegistry.def"
1598
1599   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1600 }
1601
1602 template <typename CallbacksT>
1603 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
1604   // Explicitly handle pass manager names.
1605   if (Name == "cgscc")
1606     return true;
1607   if (Name == "function")
1608     return true;
1609
1610   // Explicitly handle custom-parsed pass names.
1611   if (parseRepeatPassName(Name))
1612     return true;
1613   if (parseDevirtPassName(Name))
1614     return true;
1615
1616 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1617   if (Name == NAME)                                                            \
1618     return true;
1619 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1620   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1621     return true;
1622 #include "PassRegistry.def"
1623
1624   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
1625 }
1626
1627 template <typename CallbacksT>
1628 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
1629   // Explicitly handle pass manager names.
1630   if (Name == "function")
1631     return true;
1632   if (Name == "loop")
1633     return true;
1634
1635   // Explicitly handle custom-parsed pass names.
1636   if (parseRepeatPassName(Name))
1637     return true;
1638
1639 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1640   if (Name == NAME)                                                            \
1641     return true;
1642 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1643   if (checkParametrizedPassName(Name, NAME))                                   \
1644     return true;
1645 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1646   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1647     return true;
1648 #include "PassRegistry.def"
1649
1650   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
1651 }
1652
1653 template <typename CallbacksT>
1654 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
1655   // Explicitly handle pass manager names.
1656   if (Name == "loop")
1657     return true;
1658
1659   // Explicitly handle custom-parsed pass names.
1660   if (parseRepeatPassName(Name))
1661     return true;
1662
1663 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
1664   if (Name == NAME)                                                            \
1665     return true;
1666 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
1667   if (checkParametrizedPassName(Name, NAME))                                   \
1668     return true;
1669 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
1670   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1671     return true;
1672 #include "PassRegistry.def"
1673
1674   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
1675 }
1676
1677 Optional<std::vector<PassBuilder::PipelineElement>>
1678 PassBuilder::parsePipelineText(StringRef Text) {
1679   std::vector<PipelineElement> ResultPipeline;
1680
1681   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1682       &ResultPipeline};
1683   for (;;) {
1684     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1685     size_t Pos = Text.find_first_of(",()");
1686     Pipeline.push_back({Text.substr(0, Pos), {}});
1687
1688     // If we have a single terminating name, we're done.
1689     if (Pos == Text.npos)
1690       break;
1691
1692     char Sep = Text[Pos];
1693     Text = Text.substr(Pos + 1);
1694     if (Sep == ',')
1695       // Just a name ending in a comma, continue.
1696       continue;
1697
1698     if (Sep == '(') {
1699       // Push the inner pipeline onto the stack to continue processing.
1700       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1701       continue;
1702     }
1703
1704     assert(Sep == ')' && "Bogus separator!");
1705     // When handling the close parenthesis, we greedily consume them to avoid
1706     // empty strings in the pipeline.
1707     do {
1708       // If we try to pop the outer pipeline we have unbalanced parentheses.
1709       if (PipelineStack.size() == 1)
1710         return None;
1711
1712       PipelineStack.pop_back();
1713     } while (Text.consume_front(")"));
1714
1715     // Check if we've finished parsing.
1716     if (Text.empty())
1717       break;
1718
1719     // Otherwise, the end of an inner pipeline always has to be followed by
1720     // a comma, and then we can continue.
1721     if (!Text.consume_front(","))
1722       return None;
1723   }
1724
1725   if (PipelineStack.size() > 1)
1726     // Unbalanced paretheses.
1727     return None;
1728
1729   assert(PipelineStack.back() == &ResultPipeline &&
1730          "Wrong pipeline at the bottom of the stack!");
1731   return {std::move(ResultPipeline)};
1732 }
1733
1734 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1735                                    const PipelineElement &E,
1736                                    bool VerifyEachPass, bool DebugLogging) {
1737   auto &Name = E.Name;
1738   auto &InnerPipeline = E.InnerPipeline;
1739
1740   // First handle complex passes like the pass managers which carry pipelines.
1741   if (!InnerPipeline.empty()) {
1742     if (Name == "module") {
1743       ModulePassManager NestedMPM(DebugLogging);
1744       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1745                                              VerifyEachPass, DebugLogging))
1746         return Err;
1747       MPM.addPass(std::move(NestedMPM));
1748       return Error::success();
1749     }
1750     if (Name == "cgscc") {
1751       CGSCCPassManager CGPM(DebugLogging);
1752       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1753                                             DebugLogging))
1754         return Err;
1755       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1756       return Error::success();
1757     }
1758     if (Name == "function") {
1759       FunctionPassManager FPM(DebugLogging);
1760       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1761                                                VerifyEachPass, DebugLogging))
1762         return Err;
1763       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1764       return Error::success();
1765     }
1766     if (auto Count = parseRepeatPassName(Name)) {
1767       ModulePassManager NestedMPM(DebugLogging);
1768       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1769                                              VerifyEachPass, DebugLogging))
1770         return Err;
1771       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
1772       return Error::success();
1773     }
1774
1775     for (auto &C : ModulePipelineParsingCallbacks)
1776       if (C(Name, MPM, InnerPipeline))
1777         return Error::success();
1778
1779     // Normal passes can't have pipelines.
1780     return make_error<StringError>(
1781         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1782         inconvertibleErrorCode());
1783     ;
1784   }
1785
1786   // Manually handle aliases for pre-configured pipeline fragments.
1787   if (startsWithDefaultPipelineAliasPrefix(Name)) {
1788     SmallVector<StringRef, 3> Matches;
1789     if (!DefaultAliasRegex.match(Name, &Matches))
1790       return make_error<StringError>(
1791           formatv("unknown default pipeline alias '{0}'", Name).str(),
1792           inconvertibleErrorCode());
1793
1794     assert(Matches.size() == 3 && "Must capture two matched strings!");
1795
1796     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
1797                               .Case("O0", O0)
1798                               .Case("O1", O1)
1799                               .Case("O2", O2)
1800                               .Case("O3", O3)
1801                               .Case("Os", Os)
1802                               .Case("Oz", Oz);
1803     if (L == O0)
1804       // At O0 we do nothing at all!
1805       return Error::success();
1806
1807     if (Matches[1] == "default") {
1808       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
1809     } else if (Matches[1] == "thinlto-pre-link") {
1810       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1811     } else if (Matches[1] == "thinlto") {
1812       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
1813     } else if (Matches[1] == "lto-pre-link") {
1814       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
1815     } else {
1816       assert(Matches[1] == "lto" && "Not one of the matched options!");
1817       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
1818     }
1819     return Error::success();
1820   }
1821
1822   // Finally expand the basic registered passes from the .inc file.
1823 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1824   if (Name == NAME) {                                                          \
1825     MPM.addPass(CREATE_PASS);                                                  \
1826     return Error::success();                                                   \
1827   }
1828 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1829   if (Name == "require<" NAME ">") {                                           \
1830     MPM.addPass(                                                               \
1831         RequireAnalysisPass<                                                   \
1832             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
1833     return Error::success();                                                   \
1834   }                                                                            \
1835   if (Name == "invalidate<" NAME ">") {                                        \
1836     MPM.addPass(InvalidateAnalysisPass<                                        \
1837                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
1838     return Error::success();                                                   \
1839   }
1840 #include "PassRegistry.def"
1841
1842   for (auto &C : ModulePipelineParsingCallbacks)
1843     if (C(Name, MPM, InnerPipeline))
1844       return Error::success();
1845   return make_error<StringError>(
1846       formatv("unknown module pass '{0}'", Name).str(),
1847       inconvertibleErrorCode());
1848 }
1849
1850 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1851                                   const PipelineElement &E, bool VerifyEachPass,
1852                                   bool DebugLogging) {
1853   auto &Name = E.Name;
1854   auto &InnerPipeline = E.InnerPipeline;
1855
1856   // First handle complex passes like the pass managers which carry pipelines.
1857   if (!InnerPipeline.empty()) {
1858     if (Name == "cgscc") {
1859       CGSCCPassManager NestedCGPM(DebugLogging);
1860       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1861                                             VerifyEachPass, DebugLogging))
1862         return Err;
1863       // Add the nested pass manager with the appropriate adaptor.
1864       CGPM.addPass(std::move(NestedCGPM));
1865       return Error::success();
1866     }
1867     if (Name == "function") {
1868       FunctionPassManager FPM(DebugLogging);
1869       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1870                                                VerifyEachPass, DebugLogging))
1871         return Err;
1872       // Add the nested pass manager with the appropriate adaptor.
1873       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
1874       return Error::success();
1875     }
1876     if (auto Count = parseRepeatPassName(Name)) {
1877       CGSCCPassManager NestedCGPM(DebugLogging);
1878       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1879                                             VerifyEachPass, DebugLogging))
1880         return Err;
1881       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
1882       return Error::success();
1883     }
1884     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1885       CGSCCPassManager NestedCGPM(DebugLogging);
1886       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1887                                             VerifyEachPass, DebugLogging))
1888         return Err;
1889       CGPM.addPass(
1890           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
1891       return Error::success();
1892     }
1893
1894     for (auto &C : CGSCCPipelineParsingCallbacks)
1895       if (C(Name, CGPM, InnerPipeline))
1896         return Error::success();
1897
1898     // Normal passes can't have pipelines.
1899     return make_error<StringError>(
1900         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
1901         inconvertibleErrorCode());
1902   }
1903
1904 // Now expand the basic registered passes from the .inc file.
1905 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1906   if (Name == NAME) {                                                          \
1907     CGPM.addPass(CREATE_PASS);                                                 \
1908     return Error::success();                                                   \
1909   }
1910 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1911   if (Name == "require<" NAME ">") {                                           \
1912     CGPM.addPass(RequireAnalysisPass<                                          \
1913                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
1914                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
1915                  CGSCCUpdateResult &>());                                      \
1916     return Error::success();                                                   \
1917   }                                                                            \
1918   if (Name == "invalidate<" NAME ">") {                                        \
1919     CGPM.addPass(InvalidateAnalysisPass<                                       \
1920                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
1921     return Error::success();                                                   \
1922   }
1923 #include "PassRegistry.def"
1924
1925   for (auto &C : CGSCCPipelineParsingCallbacks)
1926     if (C(Name, CGPM, InnerPipeline))
1927       return Error::success();
1928   return make_error<StringError>(
1929       formatv("unknown cgscc pass '{0}'", Name).str(),
1930       inconvertibleErrorCode());
1931 }
1932
1933 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
1934                                      const PipelineElement &E,
1935                                      bool VerifyEachPass, bool DebugLogging) {
1936   auto &Name = E.Name;
1937   auto &InnerPipeline = E.InnerPipeline;
1938
1939   // First handle complex passes like the pass managers which carry pipelines.
1940   if (!InnerPipeline.empty()) {
1941     if (Name == "function") {
1942       FunctionPassManager NestedFPM(DebugLogging);
1943       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1944                                                VerifyEachPass, DebugLogging))
1945         return Err;
1946       // Add the nested pass manager with the appropriate adaptor.
1947       FPM.addPass(std::move(NestedFPM));
1948       return Error::success();
1949     }
1950     if (Name == "loop") {
1951       LoopPassManager LPM(DebugLogging);
1952       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
1953                                            DebugLogging))
1954         return Err;
1955       // Add the nested pass manager with the appropriate adaptor.
1956       FPM.addPass(
1957           createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging));
1958       return Error::success();
1959     }
1960     if (auto Count = parseRepeatPassName(Name)) {
1961       FunctionPassManager NestedFPM(DebugLogging);
1962       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1963                                                VerifyEachPass, DebugLogging))
1964         return Err;
1965       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
1966       return Error::success();
1967     }
1968
1969     for (auto &C : FunctionPipelineParsingCallbacks)
1970       if (C(Name, FPM, InnerPipeline))
1971         return Error::success();
1972
1973     // Normal passes can't have pipelines.
1974     return make_error<StringError>(
1975         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
1976         inconvertibleErrorCode());
1977   }
1978
1979 // Now expand the basic registered passes from the .inc file.
1980 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1981   if (Name == NAME) {                                                          \
1982     FPM.addPass(CREATE_PASS);                                                  \
1983     return Error::success();                                                   \
1984   }
1985 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1986   if (checkParametrizedPassName(Name, NAME)) {                                 \
1987     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
1988     if (!Params)                                                               \
1989       return Params.takeError();                                               \
1990     FPM.addPass(CREATE_PASS(Params.get()));                                    \
1991     return Error::success();                                                   \
1992   }
1993 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1994   if (Name == "require<" NAME ">") {                                           \
1995     FPM.addPass(                                                               \
1996         RequireAnalysisPass<                                                   \
1997             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
1998     return Error::success();                                                   \
1999   }                                                                            \
2000   if (Name == "invalidate<" NAME ">") {                                        \
2001     FPM.addPass(InvalidateAnalysisPass<                                        \
2002                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2003     return Error::success();                                                   \
2004   }
2005 #include "PassRegistry.def"
2006
2007   for (auto &C : FunctionPipelineParsingCallbacks)
2008     if (C(Name, FPM, InnerPipeline))
2009       return Error::success();
2010   return make_error<StringError>(
2011       formatv("unknown function pass '{0}'", Name).str(),
2012       inconvertibleErrorCode());
2013 }
2014
2015 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2016                                  bool VerifyEachPass, bool DebugLogging) {
2017   StringRef Name = E.Name;
2018   auto &InnerPipeline = E.InnerPipeline;
2019
2020   // First handle complex passes like the pass managers which carry pipelines.
2021   if (!InnerPipeline.empty()) {
2022     if (Name == "loop") {
2023       LoopPassManager NestedLPM(DebugLogging);
2024       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2025                                            VerifyEachPass, DebugLogging))
2026         return Err;
2027       // Add the nested pass manager with the appropriate adaptor.
2028       LPM.addPass(std::move(NestedLPM));
2029       return Error::success();
2030     }
2031     if (auto Count = parseRepeatPassName(Name)) {
2032       LoopPassManager NestedLPM(DebugLogging);
2033       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2034                                            VerifyEachPass, DebugLogging))
2035         return Err;
2036       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2037       return Error::success();
2038     }
2039
2040     for (auto &C : LoopPipelineParsingCallbacks)
2041       if (C(Name, LPM, InnerPipeline))
2042         return Error::success();
2043
2044     // Normal passes can't have pipelines.
2045     return make_error<StringError>(
2046         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2047         inconvertibleErrorCode());
2048   }
2049
2050 // Now expand the basic registered passes from the .inc file.
2051 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2052   if (Name == NAME) {                                                          \
2053     LPM.addPass(CREATE_PASS);                                                  \
2054     return Error::success();                                                   \
2055   }
2056 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2057   if (checkParametrizedPassName(Name, NAME)) {                                 \
2058     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2059     if (!Params)                                                               \
2060       return Params.takeError();                                               \
2061     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2062     return Error::success();                                                   \
2063   }
2064 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2065   if (Name == "require<" NAME ">") {                                           \
2066     LPM.addPass(RequireAnalysisPass<                                           \
2067                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2068                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2069                 LPMUpdater &>());                                              \
2070     return Error::success();                                                   \
2071   }                                                                            \
2072   if (Name == "invalidate<" NAME ">") {                                        \
2073     LPM.addPass(InvalidateAnalysisPass<                                        \
2074                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2075     return Error::success();                                                   \
2076   }
2077 #include "PassRegistry.def"
2078
2079   for (auto &C : LoopPipelineParsingCallbacks)
2080     if (C(Name, LPM, InnerPipeline))
2081       return Error::success();
2082   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2083                                  inconvertibleErrorCode());
2084 }
2085
2086 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2087 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2088   if (Name == NAME) {                                                          \
2089     AA.registerModuleAnalysis<                                                 \
2090         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2091     return true;                                                               \
2092   }
2093 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2094   if (Name == NAME) {                                                          \
2095     AA.registerFunctionAnalysis<                                               \
2096         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2097     return true;                                                               \
2098   }
2099 #include "PassRegistry.def"
2100
2101   for (auto &C : AAParsingCallbacks)
2102     if (C(Name, AA))
2103       return true;
2104   return false;
2105 }
2106
2107 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2108                                          ArrayRef<PipelineElement> Pipeline,
2109                                          bool VerifyEachPass,
2110                                          bool DebugLogging) {
2111   for (const auto &Element : Pipeline) {
2112     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2113       return Err;
2114     // FIXME: No verifier support for Loop passes!
2115   }
2116   return Error::success();
2117 }
2118
2119 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2120                                              ArrayRef<PipelineElement> Pipeline,
2121                                              bool VerifyEachPass,
2122                                              bool DebugLogging) {
2123   for (const auto &Element : Pipeline) {
2124     if (auto Err =
2125             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2126       return Err;
2127     if (VerifyEachPass)
2128       FPM.addPass(VerifierPass());
2129   }
2130   return Error::success();
2131 }
2132
2133 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2134                                           ArrayRef<PipelineElement> Pipeline,
2135                                           bool VerifyEachPass,
2136                                           bool DebugLogging) {
2137   for (const auto &Element : Pipeline) {
2138     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2139       return Err;
2140     // FIXME: No verifier support for CGSCC passes!
2141   }
2142   return Error::success();
2143 }
2144
2145 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2146                                        FunctionAnalysisManager &FAM,
2147                                        CGSCCAnalysisManager &CGAM,
2148                                        ModuleAnalysisManager &MAM) {
2149   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2150   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2151   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2152   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2153   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2154   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2155   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2156 }
2157
2158 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2159                                            ArrayRef<PipelineElement> Pipeline,
2160                                            bool VerifyEachPass,
2161                                            bool DebugLogging) {
2162   for (const auto &Element : Pipeline) {
2163     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2164       return Err;
2165     if (VerifyEachPass)
2166       MPM.addPass(VerifierPass());
2167   }
2168   return Error::success();
2169 }
2170
2171 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2172 // FIXME: Should this routine accept a TargetMachine or require the caller to
2173 // pre-populate the analysis managers with target-specific stuff?
2174 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2175                                      StringRef PipelineText,
2176                                      bool VerifyEachPass, bool DebugLogging) {
2177   auto Pipeline = parsePipelineText(PipelineText);
2178   if (!Pipeline || Pipeline->empty())
2179     return make_error<StringError>(
2180         formatv("invalid pipeline '{0}'", PipelineText).str(),
2181         inconvertibleErrorCode());
2182
2183   // If the first name isn't at the module layer, wrap the pipeline up
2184   // automatically.
2185   StringRef FirstName = Pipeline->front().Name;
2186
2187   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2188     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2189       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2190     } else if (isFunctionPassName(FirstName,
2191                                   FunctionPipelineParsingCallbacks)) {
2192       Pipeline = {{"function", std::move(*Pipeline)}};
2193     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2194       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2195     } else {
2196       for (auto &C : TopLevelPipelineParsingCallbacks)
2197         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2198           return Error::success();
2199
2200       // Unknown pass or pipeline name!
2201       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2202       return make_error<StringError>(
2203           formatv("unknown {0} name '{1}'",
2204                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2205               .str(),
2206           inconvertibleErrorCode());
2207     }
2208   }
2209
2210   if (auto Err =
2211           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2212     return Err;
2213   return Error::success();
2214 }
2215
2216 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2217 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2218                                      StringRef PipelineText,
2219                                      bool VerifyEachPass, bool DebugLogging) {
2220   auto Pipeline = parsePipelineText(PipelineText);
2221   if (!Pipeline || Pipeline->empty())
2222     return make_error<StringError>(
2223         formatv("invalid pipeline '{0}'", PipelineText).str(),
2224         inconvertibleErrorCode());
2225
2226   StringRef FirstName = Pipeline->front().Name;
2227   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2228     return make_error<StringError>(
2229         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2230                 PipelineText)
2231             .str(),
2232         inconvertibleErrorCode());
2233
2234   if (auto Err =
2235           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2236     return Err;
2237   return Error::success();
2238 }
2239
2240 // Primary pass pipeline description parsing routine for a \c
2241 // FunctionPassManager
2242 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2243                                      StringRef PipelineText,
2244                                      bool VerifyEachPass, bool DebugLogging) {
2245   auto Pipeline = parsePipelineText(PipelineText);
2246   if (!Pipeline || Pipeline->empty())
2247     return make_error<StringError>(
2248         formatv("invalid pipeline '{0}'", PipelineText).str(),
2249         inconvertibleErrorCode());
2250
2251   StringRef FirstName = Pipeline->front().Name;
2252   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2253     return make_error<StringError>(
2254         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2255                 PipelineText)
2256             .str(),
2257         inconvertibleErrorCode());
2258
2259   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2260                                            DebugLogging))
2261     return Err;
2262   return Error::success();
2263 }
2264
2265 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2266 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2267                                      StringRef PipelineText,
2268                                      bool VerifyEachPass, bool DebugLogging) {
2269   auto Pipeline = parsePipelineText(PipelineText);
2270   if (!Pipeline || Pipeline->empty())
2271     return make_error<StringError>(
2272         formatv("invalid pipeline '{0}'", PipelineText).str(),
2273         inconvertibleErrorCode());
2274
2275   if (auto Err =
2276           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2277     return Err;
2278
2279   return Error::success();
2280 }
2281
2282 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2283   // If the pipeline just consists of the word 'default' just replace the AA
2284   // manager with our default one.
2285   if (PipelineText == "default") {
2286     AA = buildDefaultAAPipeline();
2287     return Error::success();
2288   }
2289
2290   while (!PipelineText.empty()) {
2291     StringRef Name;
2292     std::tie(Name, PipelineText) = PipelineText.split(',');
2293     if (!parseAAPassName(AA, Name))
2294       return make_error<StringError>(
2295           formatv("unknown alias analysis name '{0}'", Name).str(),
2296           inconvertibleErrorCode());
2297   }
2298
2299   return Error::success();
2300 }