1 //===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This pass builds a ModuleSummaryIndex object for the module, to be written
10 // to bitcode or LLVM assembly.
12 //===----------------------------------------------------------------------===//
14 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/DenseSet.h"
17 #include "llvm/ADT/MapVector.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/ADT/SetVector.h"
20 #include "llvm/ADT/SmallPtrSet.h"
21 #include "llvm/ADT/SmallVector.h"
22 #include "llvm/ADT/StringRef.h"
23 #include "llvm/Analysis/BlockFrequencyInfo.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
26 #include "llvm/Analysis/LoopInfo.h"
27 #include "llvm/Analysis/ProfileSummaryInfo.h"
28 #include "llvm/Analysis/TypeMetadataUtils.h"
29 #include "llvm/IR/Attributes.h"
30 #include "llvm/IR/BasicBlock.h"
31 #include "llvm/IR/CallSite.h"
32 #include "llvm/IR/Constant.h"
33 #include "llvm/IR/Constants.h"
34 #include "llvm/IR/Dominators.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/GlobalAlias.h"
37 #include "llvm/IR/GlobalValue.h"
38 #include "llvm/IR/GlobalVariable.h"
39 #include "llvm/IR/Instructions.h"
40 #include "llvm/IR/IntrinsicInst.h"
41 #include "llvm/IR/Intrinsics.h"
42 #include "llvm/IR/Metadata.h"
43 #include "llvm/IR/Module.h"
44 #include "llvm/IR/ModuleSummaryIndex.h"
45 #include "llvm/IR/Use.h"
46 #include "llvm/IR/User.h"
47 #include "llvm/Object/ModuleSymbolTable.h"
48 #include "llvm/Object/SymbolicFile.h"
49 #include "llvm/Pass.h"
50 #include "llvm/Support/Casting.h"
51 #include "llvm/Support/CommandLine.h"
59 #define DEBUG_TYPE "module-summary-analysis"
61 // Option to force edges cold which will block importing when the
62 // -import-cold-multiplier is set to 0. Useful for debugging.
63 FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold =
64 FunctionSummary::FSHT_None;
65 cl::opt<FunctionSummary::ForceSummaryHotnessType, true> FSEC(
66 "force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
67 cl::desc("Force all edges in the function summary to cold"),
68 cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."),
69 clEnumValN(FunctionSummary::FSHT_AllNonCritical,
70 "all-non-critical", "All non-critical edges."),
71 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
73 cl::opt<std::string> ModuleSummaryDotFile(
74 "module-summary-dot-file", cl::init(""), cl::Hidden,
75 cl::value_desc("filename"),
76 cl::desc("File to emit dot graph of new summary into."));
78 // Walk through the operands of a given User via worklist iteration and populate
79 // the set of GlobalValue references encountered. Invoked either on an
80 // Instruction or a GlobalVariable (which walks its initializer).
81 // Return true if any of the operands contains blockaddress. This is important
82 // to know when computing summary for global var, because if global variable
83 // references basic block address we can't import it separately from function
84 // containing that basic block. For simplicity we currently don't import such
85 // global vars at all. When importing function we aren't interested if any
86 // instruction in it takes an address of any basic block, because instruction
87 // can only take an address of basic block located in the same function.
88 static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
89 SetVector<ValueInfo> &RefEdges,
90 SmallPtrSet<const User *, 8> &Visited) {
91 bool HasBlockAddress = false;
92 SmallVector<const User *, 32> Worklist;
93 Worklist.push_back(CurUser);
95 while (!Worklist.empty()) {
96 const User *U = Worklist.pop_back_val();
98 if (!Visited.insert(U).second)
101 ImmutableCallSite CS(U);
103 for (const auto &OI : U->operands()) {
104 const User *Operand = dyn_cast<User>(OI);
107 if (isa<BlockAddress>(Operand)) {
108 HasBlockAddress = true;
111 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
112 // We have a reference to a global value. This should be added to
113 // the reference set unless it is a callee. Callees are handled
114 // specially by WriteFunction and are added to a separate list.
115 if (!(CS && CS.isCallee(&OI)))
116 RefEdges.insert(Index.getOrInsertValueInfo(GV));
119 Worklist.push_back(Operand);
122 return HasBlockAddress;
125 static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount,
126 ProfileSummaryInfo *PSI) {
128 return CalleeInfo::HotnessType::Unknown;
129 if (PSI->isHotCount(ProfileCount))
130 return CalleeInfo::HotnessType::Hot;
131 if (PSI->isColdCount(ProfileCount))
132 return CalleeInfo::HotnessType::Cold;
133 return CalleeInfo::HotnessType::None;
136 static bool isNonRenamableLocal(const GlobalValue &GV) {
137 return GV.hasSection() && GV.hasLocalLinkage();
140 /// Determine whether this call has all constant integer arguments (excluding
141 /// "this") and summarize it to VCalls or ConstVCalls as appropriate.
142 static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid,
143 SetVector<FunctionSummary::VFuncId> &VCalls,
144 SetVector<FunctionSummary::ConstVCall> &ConstVCalls) {
145 std::vector<uint64_t> Args;
146 // Start from the second argument to skip the "this" pointer.
147 for (auto &Arg : make_range(Call.CS.arg_begin() + 1, Call.CS.arg_end())) {
148 auto *CI = dyn_cast<ConstantInt>(Arg);
149 if (!CI || CI->getBitWidth() > 64) {
150 VCalls.insert({Guid, Call.Offset});
153 Args.push_back(CI->getZExtValue());
155 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
158 /// If this intrinsic call requires that we add information to the function
159 /// summary, do so via the non-constant reference arguments.
160 static void addIntrinsicToSummary(
161 const CallInst *CI, SetVector<GlobalValue::GUID> &TypeTests,
162 SetVector<FunctionSummary::VFuncId> &TypeTestAssumeVCalls,
163 SetVector<FunctionSummary::VFuncId> &TypeCheckedLoadVCalls,
164 SetVector<FunctionSummary::ConstVCall> &TypeTestAssumeConstVCalls,
165 SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls,
167 switch (CI->getCalledFunction()->getIntrinsicID()) {
168 case Intrinsic::type_test: {
169 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
170 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
173 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
175 // Produce a summary from type.test intrinsics. We only summarize type.test
176 // intrinsics that are used other than by an llvm.assume intrinsic.
177 // Intrinsics that are assumed are relevant only to the devirtualization
178 // pass, not the type test lowering pass.
179 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
180 auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
183 Function *F = AssumeCI->getCalledFunction();
184 return !F || F->getIntrinsicID() != Intrinsic::assume;
186 if (HasNonAssumeUses)
187 TypeTests.insert(Guid);
189 SmallVector<DevirtCallSite, 4> DevirtCalls;
190 SmallVector<CallInst *, 4> Assumes;
191 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
192 for (auto &Call : DevirtCalls)
193 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
194 TypeTestAssumeConstVCalls);
199 case Intrinsic::type_checked_load: {
200 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
201 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
204 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
206 SmallVector<DevirtCallSite, 4> DevirtCalls;
207 SmallVector<Instruction *, 4> LoadedPtrs;
208 SmallVector<Instruction *, 4> Preds;
209 bool HasNonCallUses = false;
210 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
211 HasNonCallUses, CI, DT);
212 // Any non-call uses of the result of llvm.type.checked.load will
213 // prevent us from optimizing away the llvm.type.test.
215 TypeTests.insert(Guid);
216 for (auto &Call : DevirtCalls)
217 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
218 TypeCheckedLoadConstVCalls);
227 static bool isNonVolatileLoad(const Instruction *I) {
228 if (const auto *LI = dyn_cast<LoadInst>(I))
229 return !LI->isVolatile();
234 static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
235 const Function &F, BlockFrequencyInfo *BFI,
236 ProfileSummaryInfo *PSI, DominatorTree &DT,
237 bool HasLocalsInUsedOrAsm,
238 DenseSet<GlobalValue::GUID> &CantBePromoted,
240 // Summary not currently supported for anonymous functions, they should
244 unsigned NumInsts = 0;
245 // Map from callee ValueId to profile count. Used to accumulate profile
246 // counts for all static calls to a given callee.
247 MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
248 SetVector<ValueInfo> RefEdges;
249 SetVector<GlobalValue::GUID> TypeTests;
250 SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
251 TypeCheckedLoadVCalls;
252 SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
253 TypeCheckedLoadConstVCalls;
254 ICallPromotionAnalysis ICallAnalysis;
255 SmallPtrSet<const User *, 8> Visited;
257 // Add personality function, prefix data and prologue data to function's ref
259 findRefEdges(Index, &F, RefEdges, Visited);
260 std::vector<const Instruction *> NonVolatileLoads;
262 bool HasInlineAsmMaybeReferencingInternal = false;
263 for (const BasicBlock &BB : F)
264 for (const Instruction &I : BB) {
265 if (isa<DbgInfoIntrinsic>(I))
268 if (isNonVolatileLoad(&I)) {
269 // Postpone processing of non-volatile load instructions
270 // See comments below
272 NonVolatileLoads.push_back(&I);
275 findRefEdges(Index, &I, RefEdges, Visited);
276 auto CS = ImmutableCallSite(&I);
280 const auto *CI = dyn_cast<CallInst>(&I);
281 // Since we don't know exactly which local values are referenced in inline
282 // assembly, conservatively mark the function as possibly referencing
283 // a local value from inline assembly to ensure we don't export a
284 // reference (which would require renaming and promotion of the
285 // referenced value).
286 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
287 HasInlineAsmMaybeReferencingInternal = true;
289 auto *CalledValue = CS.getCalledValue();
290 auto *CalledFunction = CS.getCalledFunction();
291 if (CalledValue && !CalledFunction) {
292 CalledValue = CalledValue->stripPointerCastsNoFollowAliases();
293 // Stripping pointer casts can reveal a called function.
294 CalledFunction = dyn_cast<Function>(CalledValue);
296 // Check if this is an alias to a function. If so, get the
297 // called aliasee for the checks below.
298 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
299 assert(!CalledFunction && "Expected null called function in callsite for alias");
300 CalledFunction = dyn_cast<Function>(GA->getBaseObject());
302 // Check if this is a direct call to a known function or a known
303 // intrinsic, or an indirect call with profile data.
304 if (CalledFunction) {
305 if (CI && CalledFunction->isIntrinsic()) {
306 addIntrinsicToSummary(
307 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
308 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
311 // We should have named any anonymous globals
312 assert(CalledFunction->hasName());
313 auto ScaledCount = PSI->getProfileCount(&I, BFI);
314 auto Hotness = ScaledCount ? getHotness(ScaledCount.getValue(), PSI)
315 : CalleeInfo::HotnessType::Unknown;
316 if (ForceSummaryEdgesCold != FunctionSummary::FSHT_None)
317 Hotness = CalleeInfo::HotnessType::Cold;
319 // Use the original CalledValue, in case it was an alias. We want
320 // to record the call edge to the alias in that case. Eventually
321 // an alias summary will be created to associate the alias and
323 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
324 cast<GlobalValue>(CalledValue))];
325 ValueInfo.updateHotness(Hotness);
326 // Add the relative block frequency to CalleeInfo if there is no profile
328 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
329 uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
330 uint64_t EntryFreq = BFI->getEntryFreq();
331 ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
334 // Skip inline assembly calls.
335 if (CI && CI->isInlineAsm())
337 // Skip direct calls.
338 if (!CalledValue || isa<Constant>(CalledValue))
341 // Check if the instruction has a callees metadata. If so, add callees
342 // to CallGraphEdges to reflect the references from the metadata, and
343 // to enable importing for subsequent indirect call promotion and
345 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
346 for (auto &Op : MD->operands()) {
347 Function *Callee = mdconst::extract_or_null<Function>(Op);
349 CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
353 uint32_t NumVals, NumCandidates;
355 auto CandidateProfileData =
356 ICallAnalysis.getPromotionCandidatesForInstruction(
357 &I, NumVals, TotalCount, NumCandidates);
358 for (auto &Candidate : CandidateProfileData)
359 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
360 .updateHotness(getHotness(Candidate.Count, PSI));
364 // By now we processed all instructions in a function, except
365 // non-volatile loads. All new refs we add in a loop below
366 // are obviously constant. All constant refs are grouped in the
367 // end of RefEdges vector, so we can use a single integer value
369 unsigned RefCnt = RefEdges.size();
370 for (const Instruction *I : NonVolatileLoads) {
372 findRefEdges(Index, I, RefEdges, Visited);
374 std::vector<ValueInfo> Refs = RefEdges.takeVector();
375 // Regular LTO module doesn't participate in ThinLTO import,
376 // so no reference from it can be readonly, since this would
377 // require importing variable as local copy
379 for (; RefCnt < Refs.size(); ++RefCnt)
380 Refs[RefCnt].setReadOnly();
382 // Explicit add hot edges to enforce importing for designated GUIDs for
383 // sample PGO, to enable the same inlines as the profiled optimized binary.
384 for (auto &I : F.getImportGUIDs())
385 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
386 ForceSummaryEdgesCold == FunctionSummary::FSHT_All
387 ? CalleeInfo::HotnessType::Cold
388 : CalleeInfo::HotnessType::Critical);
390 bool NonRenamableLocal = isNonRenamableLocal(F);
391 bool NotEligibleForImport =
392 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal;
393 GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport,
394 /* Live = */ false, F.isDSOLocal(),
395 F.hasLinkOnceODRLinkage() && F.hasGlobalUnnamedAddr());
396 FunctionSummary::FFlags FunFlags{
397 F.hasFnAttribute(Attribute::ReadNone),
398 F.hasFnAttribute(Attribute::ReadOnly),
399 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
400 // FIXME: refactor this to use the same code that inliner is using.
401 // Don't try to import functions with noinline attribute.
402 F.getAttributes().hasFnAttribute(Attribute::NoInline)};
403 auto FuncSummary = llvm::make_unique<FunctionSummary>(
404 Flags, NumInsts, FunFlags, /*EntryCount=*/0, std::move(Refs),
405 CallGraphEdges.takeVector(), TypeTests.takeVector(),
406 TypeTestAssumeVCalls.takeVector(), TypeCheckedLoadVCalls.takeVector(),
407 TypeTestAssumeConstVCalls.takeVector(),
408 TypeCheckedLoadConstVCalls.takeVector());
409 if (NonRenamableLocal)
410 CantBePromoted.insert(F.getGUID());
411 Index.addGlobalValueSummary(F, std::move(FuncSummary));
414 /// Find function pointers referenced within the given vtable initializer
415 /// (or subset of an initializer) \p I. The starting offset of \p I within
416 /// the vtable initializer is \p StartingOffset. Any discovered function
417 /// pointers are added to \p VTableFuncs along with their cumulative offset
418 /// within the initializer.
419 static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
420 const Module &M, ModuleSummaryIndex &Index,
421 VTableFuncList &VTableFuncs) {
422 // First check if this is a function pointer.
423 if (I->getType()->isPointerTy()) {
424 auto Fn = dyn_cast<Function>(I->stripPointerCasts());
425 // We can disregard __cxa_pure_virtual as a possible call target, as
426 // calls to pure virtuals are UB.
427 if (Fn && Fn->getName() != "__cxa_pure_virtual")
428 VTableFuncs.push_back({Index.getOrInsertValueInfo(Fn), StartingOffset});
432 // Walk through the elements in the constant struct or array and recursively
433 // look for virtual function pointers.
434 const DataLayout &DL = M.getDataLayout();
435 if (auto *C = dyn_cast<ConstantStruct>(I)) {
436 StructType *STy = dyn_cast<StructType>(C->getType());
438 const StructLayout *SL = DL.getStructLayout(C->getType());
440 for (StructType::element_iterator EB = STy->element_begin(), EI = EB,
441 EE = STy->element_end();
443 auto Offset = SL->getElementOffset(EI - EB);
444 unsigned Op = SL->getElementContainingOffset(Offset);
445 findFuncPointers(cast<Constant>(I->getOperand(Op)),
446 StartingOffset + Offset, M, Index, VTableFuncs);
448 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
449 ArrayType *ATy = C->getType();
450 Type *EltTy = ATy->getElementType();
451 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
452 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
453 findFuncPointers(cast<Constant>(I->getOperand(i)),
454 StartingOffset + i * EltSize, M, Index, VTableFuncs);
459 // Identify the function pointers referenced by vtable definition \p V.
460 static void computeVTableFuncs(ModuleSummaryIndex &Index,
461 const GlobalVariable &V, const Module &M,
462 VTableFuncList &VTableFuncs) {
466 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
470 // Validate that the VTableFuncs list is ordered by offset.
471 uint64_t PrevOffset = 0;
472 for (auto &P : VTableFuncs) {
473 // The findVFuncPointers traversal should have encountered the
474 // functions in offset order. We need to use ">=" since PrevOffset
476 assert(P.VTableOffset >= PrevOffset);
477 PrevOffset = P.VTableOffset;
482 /// Record vtable definition \p V for each type metadata it references.
484 recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index,
485 const GlobalVariable &V,
486 SmallVectorImpl<MDNode *> &Types) {
487 for (MDNode *Type : Types) {
488 auto TypeID = Type->getOperand(1).get();
492 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
495 if (auto *TypeId = dyn_cast<MDString>(TypeID))
496 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
497 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
501 static void computeVariableSummary(ModuleSummaryIndex &Index,
502 const GlobalVariable &V,
503 DenseSet<GlobalValue::GUID> &CantBePromoted,
505 SmallVectorImpl<MDNode *> &Types) {
506 SetVector<ValueInfo> RefEdges;
507 SmallPtrSet<const User *, 8> Visited;
508 bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited);
509 bool NonRenamableLocal = isNonRenamableLocal(V);
510 GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
511 /* Live = */ false, V.isDSOLocal(),
512 V.hasLinkOnceODRLinkage() && V.hasGlobalUnnamedAddr());
514 VTableFuncList VTableFuncs;
515 // If splitting is not enabled, then we compute the summary information
516 // necessary for index-based whole program devirtualization.
517 if (!Index.enableSplitLTOUnit()) {
519 V.getMetadata(LLVMContext::MD_type, Types);
520 if (!Types.empty()) {
521 // Identify the function pointers referenced by this vtable definition.
522 computeVTableFuncs(Index, V, M, VTableFuncs);
524 // Record this vtable definition for each type metadata it references.
525 recordTypeIdCompatibleVtableReferences(Index, V, Types);
529 // Don't mark variables we won't be able to internalize as read-only.
530 GlobalVarSummary::GVarFlags VarFlags(
531 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
532 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass());
533 auto GVarSummary = llvm::make_unique<GlobalVarSummary>(Flags, VarFlags,
534 RefEdges.takeVector());
535 if (NonRenamableLocal)
536 CantBePromoted.insert(V.getGUID());
538 GVarSummary->setNotEligibleToImport();
539 if (!VTableFuncs.empty())
540 GVarSummary->setVTableFuncs(VTableFuncs);
541 Index.addGlobalValueSummary(V, std::move(GVarSummary));
545 computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A,
546 DenseSet<GlobalValue::GUID> &CantBePromoted) {
547 bool NonRenamableLocal = isNonRenamableLocal(A);
548 GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal,
549 /* Live = */ false, A.isDSOLocal(),
550 A.hasLinkOnceODRLinkage() && A.hasGlobalUnnamedAddr());
551 auto AS = llvm::make_unique<AliasSummary>(Flags);
552 auto *Aliasee = A.getBaseObject();
553 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
554 assert(AliaseeVI && "Alias expects aliasee summary to be available");
555 assert(AliaseeVI.getSummaryList().size() == 1 &&
556 "Expected a single entry per aliasee in per-module index");
557 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
558 if (NonRenamableLocal)
559 CantBePromoted.insert(A.getGUID());
560 Index.addGlobalValueSummary(A, std::move(AS));
563 // Set LiveRoot flag on entries matching the given value name.
564 static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
565 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
566 for (auto &Summary : VI.getSummaryList())
567 Summary->setLive(true);
570 ModuleSummaryIndex llvm::buildModuleSummaryIndex(
572 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
573 ProfileSummaryInfo *PSI) {
575 bool EnableSplitLTOUnit = false;
576 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
577 M.getModuleFlag("EnableSplitLTOUnit")))
578 EnableSplitLTOUnit = MD->getZExtValue();
579 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit);
581 // Identify the local values in the llvm.used and llvm.compiler.used sets,
582 // which should not be exported as they would then require renaming and
583 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
584 // here because we use this information to mark functions containing inline
585 // assembly calls as not importable.
586 SmallPtrSet<GlobalValue *, 8> LocalsUsed;
587 SmallPtrSet<GlobalValue *, 8> Used;
588 // First collect those in the llvm.used set.
589 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ false);
590 // Next collect those in the llvm.compiler.used set.
591 collectUsedGlobalVariables(M, Used, /*CompilerUsed*/ true);
592 DenseSet<GlobalValue::GUID> CantBePromoted;
593 for (auto *V : Used) {
594 if (V->hasLocalLinkage()) {
595 LocalsUsed.insert(V);
596 CantBePromoted.insert(V->getGUID());
600 bool HasLocalInlineAsmSymbol = false;
601 if (!M.getModuleInlineAsm().empty()) {
602 // Collect the local values defined by module level asm, and set up
603 // summaries for these symbols so that they can be marked as NoRename,
604 // to prevent export of any use of them in regular IR that would require
605 // renaming within the module level asm. Note we don't need to create a
606 // summary for weak or global defs, as they don't need to be flagged as
607 // NoRename, and defs in module level asm can't be imported anyway.
608 // Also, any values used but not defined within module level asm should
609 // be listed on the llvm.used or llvm.compiler.used global and marked as
610 // referenced from there.
611 ModuleSymbolTable::CollectAsmSymbols(
612 M, [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
613 // Symbols not marked as Weak or Global are local definitions.
614 if (Flags & (object::BasicSymbolRef::SF_Weak |
615 object::BasicSymbolRef::SF_Global))
617 HasLocalInlineAsmSymbol = true;
618 GlobalValue *GV = M.getNamedValue(Name);
621 assert(GV->isDeclaration() && "Def in module asm already has definition");
622 GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage,
623 /* NotEligibleToImport = */ true,
625 /* Local */ GV->isDSOLocal(),
626 GV->hasLinkOnceODRLinkage() && GV->hasGlobalUnnamedAddr());
627 CantBePromoted.insert(GV->getGUID());
628 // Create the appropriate summary type.
629 if (Function *F = dyn_cast<Function>(GV)) {
630 std::unique_ptr<FunctionSummary> Summary =
631 llvm::make_unique<FunctionSummary>(
632 GVFlags, /*InstCount=*/0,
633 FunctionSummary::FFlags{
634 F->hasFnAttribute(Attribute::ReadNone),
635 F->hasFnAttribute(Attribute::ReadOnly),
636 F->hasFnAttribute(Attribute::NoRecurse),
637 F->returnDoesNotAlias(),
638 /* NoInline = */ false},
639 /*EntryCount=*/0, ArrayRef<ValueInfo>{},
640 ArrayRef<FunctionSummary::EdgeTy>{},
641 ArrayRef<GlobalValue::GUID>{},
642 ArrayRef<FunctionSummary::VFuncId>{},
643 ArrayRef<FunctionSummary::VFuncId>{},
644 ArrayRef<FunctionSummary::ConstVCall>{},
645 ArrayRef<FunctionSummary::ConstVCall>{});
646 Index.addGlobalValueSummary(*GV, std::move(Summary));
648 std::unique_ptr<GlobalVarSummary> Summary =
649 llvm::make_unique<GlobalVarSummary>(
650 GVFlags, GlobalVarSummary::GVarFlags(),
651 ArrayRef<ValueInfo>{});
652 Index.addGlobalValueSummary(*GV, std::move(Summary));
657 bool IsThinLTO = true;
659 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
660 IsThinLTO = MD->getZExtValue();
662 // Compute summaries for all functions defined in module, and save in the
665 if (F.isDeclaration())
668 DominatorTree DT(const_cast<Function &>(F));
669 BlockFrequencyInfo *BFI = nullptr;
670 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
672 BFI = GetBFICallback(F);
673 else if (F.hasProfileData()) {
675 BranchProbabilityInfo BPI{F, LI};
676 BFIPtr = llvm::make_unique<BlockFrequencyInfo>(F, BPI, LI);
680 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
681 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
682 CantBePromoted, IsThinLTO);
685 // Compute summaries for all variables defined in module, and save in the
687 SmallVector<MDNode *, 2> Types;
688 for (const GlobalVariable &G : M.globals()) {
689 if (G.isDeclaration())
691 computeVariableSummary(Index, G, CantBePromoted, M, Types);
694 // Compute summaries for all aliases defined in module, and save in the
696 for (const GlobalAlias &A : M.aliases())
697 computeAliasSummary(Index, A, CantBePromoted);
699 for (auto *V : LocalsUsed) {
700 auto *Summary = Index.getGlobalValueSummary(*V);
701 assert(Summary && "Missing summary for global value");
702 Summary->setNotEligibleToImport();
705 // The linker doesn't know about these LLVM produced values, so we need
706 // to flag them as live in the index to ensure index-based dead value
707 // analysis treats them as live roots of the analysis.
708 setLiveRoot(Index, "llvm.used");
709 setLiveRoot(Index, "llvm.compiler.used");
710 setLiveRoot(Index, "llvm.global_ctors");
711 setLiveRoot(Index, "llvm.global_dtors");
712 setLiveRoot(Index, "llvm.global.annotations");
714 for (auto &GlobalList : Index) {
715 // Ignore entries for references that are undefined in the current module.
716 if (GlobalList.second.SummaryList.empty())
719 assert(GlobalList.second.SummaryList.size() == 1 &&
720 "Expected module's index to have one summary per GUID");
721 auto &Summary = GlobalList.second.SummaryList[0];
723 Summary->setNotEligibleToImport();
727 bool AllRefsCanBeExternallyReferenced =
728 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
729 return !CantBePromoted.count(VI.getGUID());
731 if (!AllRefsCanBeExternallyReferenced) {
732 Summary->setNotEligibleToImport();
736 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
737 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
738 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
739 return !CantBePromoted.count(Edge.first.getGUID());
741 if (!AllCallsCanBeExternallyReferenced)
742 Summary->setNotEligibleToImport();
746 if (!ModuleSummaryDotFile.empty()) {
748 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::F_None);
750 report_fatal_error(Twine("Failed to open dot file ") +
751 ModuleSummaryDotFile + ": " + EC.message() + "\n");
752 Index.exportToDot(OSDot);
758 AnalysisKey ModuleSummaryIndexAnalysis::Key;
761 ModuleSummaryIndexAnalysis::run(Module &M, ModuleAnalysisManager &AM) {
762 ProfileSummaryInfo &PSI = AM.getResult<ProfileSummaryAnalysis>(M);
763 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
764 return buildModuleSummaryIndex(
766 [&FAM](const Function &F) {
767 return &FAM.getResult<BlockFrequencyAnalysis>(
768 *const_cast<Function *>(&F));
773 char ModuleSummaryIndexWrapperPass::ID = 0;
775 INITIALIZE_PASS_BEGIN(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
776 "Module Summary Analysis", false, true)
777 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
778 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
779 INITIALIZE_PASS_END(ModuleSummaryIndexWrapperPass, "module-summary-analysis",
780 "Module Summary Analysis", false, true)
782 ModulePass *llvm::createModuleSummaryIndexWrapperPass() {
783 return new ModuleSummaryIndexWrapperPass();
786 ModuleSummaryIndexWrapperPass::ModuleSummaryIndexWrapperPass()
788 initializeModuleSummaryIndexWrapperPassPass(*PassRegistry::getPassRegistry());
791 bool ModuleSummaryIndexWrapperPass::runOnModule(Module &M) {
792 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
793 Index.emplace(buildModuleSummaryIndex(
795 [this](const Function &F) {
796 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
797 *const_cast<Function *>(&F))
804 bool ModuleSummaryIndexWrapperPass::doFinalization(Module &M) {
809 void ModuleSummaryIndexWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
810 AU.setPreservesAll();
811 AU.addRequired<BlockFrequencyInfoWrapperPass>();
812 AU.addRequired<ProfileSummaryInfoWrapperPass>();