1 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file implements the Thin Link Time Optimization library. This library is
11 // intended to be used by linker to optimize code at link time.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/LTO/legacy/ThinLTOCodeGenerator.h"
17 #ifdef HAVE_LLVM_REVISION
18 #include "LLVMLTORevision.h"
21 #include "llvm/ADT/Statistic.h"
22 #include "llvm/ADT/StringExtras.h"
23 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
24 #include "llvm/Analysis/ProfileSummaryInfo.h"
25 #include "llvm/Analysis/TargetLibraryInfo.h"
26 #include "llvm/Analysis/TargetTransformInfo.h"
27 #include "llvm/Bitcode/BitcodeReader.h"
28 #include "llvm/Bitcode/BitcodeWriter.h"
29 #include "llvm/Bitcode/BitcodeWriterPass.h"
30 #include "llvm/ExecutionEngine/ObjectMemoryBuffer.h"
31 #include "llvm/IR/DiagnosticPrinter.h"
32 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/IR/LegacyPassManager.h"
34 #include "llvm/IR/Mangler.h"
35 #include "llvm/IRReader/IRReader.h"
36 #include "llvm/LTO/LTO.h"
37 #include "llvm/Linker/Linker.h"
38 #include "llvm/MC/SubtargetFeature.h"
39 #include "llvm/Object/IRObjectFile.h"
40 #include "llvm/Object/ModuleSummaryIndexObjectFile.h"
41 #include "llvm/Support/CachePruning.h"
42 #include "llvm/Support/Debug.h"
43 #include "llvm/Support/Error.h"
44 #include "llvm/Support/Path.h"
45 #include "llvm/Support/SHA1.h"
46 #include "llvm/Support/TargetRegistry.h"
47 #include "llvm/Support/ThreadPool.h"
48 #include "llvm/Support/Threading.h"
49 #include "llvm/Support/ToolOutputFile.h"
50 #include "llvm/Target/TargetMachine.h"
51 #include "llvm/Transforms/IPO.h"
52 #include "llvm/Transforms/IPO/FunctionImport.h"
53 #include "llvm/Transforms/IPO/Internalize.h"
54 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
55 #include "llvm/Transforms/ObjCARC.h"
56 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
62 #define DEBUG_TYPE "thinlto"
65 // Flags -discard-value-names, defined in LTOCodeGenerator.cpp
66 extern cl::opt<bool> LTODiscardValueNames;
67 extern cl::opt<std::string> LTORemarksFilename;
68 extern cl::opt<bool> LTOPassRemarksWithHotness;
74 ThreadCount("threads", cl::init(llvm::heavyweight_hardware_concurrency()));
76 // Simple helper to save temporary files for debug.
77 static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
78 unsigned count, StringRef Suffix) {
81 // User asked to save temps, let dump the bitcode file after import.
82 std::string SaveTempPath = (TempDir + llvm::utostr(count) + Suffix).str();
84 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
86 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
87 " to save optimized bitcode\n");
88 WriteBitcodeToFile(&TheModule, OS, /* ShouldPreserveUseListOrder */ true);
91 static const GlobalValueSummary *
92 getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) {
93 // If there is any strong definition anywhere, get it.
94 auto StrongDefForLinker = llvm::find_if(
95 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
96 auto Linkage = Summary->linkage();
97 return !GlobalValue::isAvailableExternallyLinkage(Linkage) &&
98 !GlobalValue::isWeakForLinker(Linkage);
100 if (StrongDefForLinker != GVSummaryList.end())
101 return StrongDefForLinker->get();
102 // Get the first *linker visible* definition for this global in the summary
104 auto FirstDefForLinker = llvm::find_if(
105 GVSummaryList, [](const std::unique_ptr<GlobalValueSummary> &Summary) {
106 auto Linkage = Summary->linkage();
107 return !GlobalValue::isAvailableExternallyLinkage(Linkage);
109 // Extern templates can be emitted as available_externally.
110 if (FirstDefForLinker == GVSummaryList.end())
112 return FirstDefForLinker->get();
115 // Populate map of GUID to the prevailing copy for any multiply defined
116 // symbols. Currently assume first copy is prevailing, or any strong
117 // definition. Can be refined with Linker information in the future.
118 static void computePrevailingCopies(
119 const ModuleSummaryIndex &Index,
120 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy) {
121 auto HasMultipleCopies = [&](const GlobalValueSummaryList &GVSummaryList) {
122 return GVSummaryList.size() > 1;
125 for (auto &I : Index) {
126 if (HasMultipleCopies(I.second))
127 PrevailingCopy[I.first] = getFirstDefinitionForLinker(I.second);
131 static StringMap<MemoryBufferRef>
132 generateModuleMap(const std::vector<ThinLTOBuffer> &Modules) {
133 StringMap<MemoryBufferRef> ModuleMap;
134 for (auto &ModuleBuffer : Modules) {
135 assert(ModuleMap.find(ModuleBuffer.getBufferIdentifier()) ==
137 "Expect unique Buffer Identifier");
138 ModuleMap[ModuleBuffer.getBufferIdentifier()] = ModuleBuffer.getMemBuffer();
143 static void promoteModule(Module &TheModule, const ModuleSummaryIndex &Index) {
144 if (renameModuleForThinLTO(TheModule, Index))
145 report_fatal_error("renameModuleForThinLTO failed");
148 static std::unique_ptr<Module>
149 loadModuleFromBuffer(const MemoryBufferRef &Buffer, LLVMContext &Context,
150 bool Lazy, bool IsImporting) {
152 Expected<std::unique_ptr<Module>> ModuleOrErr =
154 ? getLazyBitcodeModule(Buffer, Context,
155 /* ShouldLazyLoadMetadata */ true, IsImporting)
156 : parseBitcodeFile(Buffer, Context);
158 handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
159 SMDiagnostic Err = SMDiagnostic(Buffer.getBufferIdentifier(),
160 SourceMgr::DK_Error, EIB.message());
161 Err.print("ThinLTO", errs());
163 report_fatal_error("Can't load module, abort.");
165 return std::move(ModuleOrErr.get());
169 crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
170 StringMap<MemoryBufferRef> &ModuleMap,
171 const FunctionImporter::ImportMapTy &ImportList) {
172 auto Loader = [&](StringRef Identifier) {
173 return loadModuleFromBuffer(ModuleMap[Identifier], TheModule.getContext(),
174 /*Lazy=*/true, /*IsImporting*/ true);
177 FunctionImporter Importer(Index, Loader);
178 Expected<bool> Result = Importer.importFunctions(TheModule, ImportList);
180 handleAllErrors(Result.takeError(), [&](ErrorInfoBase &EIB) {
181 SMDiagnostic Err = SMDiagnostic(TheModule.getModuleIdentifier(),
182 SourceMgr::DK_Error, EIB.message());
183 Err.print("ThinLTO", errs());
185 report_fatal_error("importFunctions failed");
189 static void optimizeModule(Module &TheModule, TargetMachine &TM,
190 unsigned OptLevel, bool Freestanding) {
191 // Populate the PassManager
192 PassManagerBuilder PMB;
193 PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
195 PMB.LibraryInfo->disableAllFunctions();
196 PMB.Inliner = createFunctionInliningPass();
197 // FIXME: should get it from the bitcode?
198 PMB.OptLevel = OptLevel;
199 PMB.LoopVectorize = true;
200 PMB.SLPVectorize = true;
201 PMB.VerifyInput = true;
202 PMB.VerifyOutput = false;
204 legacy::PassManager PM;
206 // Add the TTI (required to inform the vectorizer about register size for
208 PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
211 PMB.populateThinLTOPassManager(PM);
216 // Convert the PreservedSymbols map from "Name" based to "GUID" based.
217 static DenseSet<GlobalValue::GUID>
218 computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
219 const Triple &TheTriple) {
220 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
221 for (auto &Entry : PreservedSymbols) {
222 StringRef Name = Entry.first();
223 if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
224 Name = Name.drop_front();
225 GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
227 return GUIDPreservedSymbols;
230 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
232 SmallVector<char, 128> OutputBuffer;
236 raw_svector_ostream OS(OutputBuffer);
237 legacy::PassManager PM;
239 // If the bitcode files contain ARC code and were compiled with optimization,
240 // the ObjCARCContractPass must be run, so do it unconditionally here.
241 PM.add(createObjCARCContractPass());
243 // Setup the codegen now.
244 if (TM.addPassesToEmitFile(PM, OS, TargetMachine::CGFT_ObjectFile,
245 /* DisableVerify */ true))
246 report_fatal_error("Failed to setup codegen");
248 // Run codegen now. resulting binary is in OutputBuffer.
251 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
254 /// Manage caching for a single Module.
255 class ModuleCacheEntry {
256 SmallString<128> EntryPath;
259 // Create a cache entry. This compute a unique hash for the Module considering
260 // the current list of export/import, and offer an interface to query to
261 // access the content in the cache.
263 StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
264 const FunctionImporter::ImportMapTy &ImportList,
265 const FunctionImporter::ExportSetTy &ExportList,
266 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
267 const GVSummaryMapTy &DefinedFunctions,
268 const DenseSet<GlobalValue::GUID> &PreservedSymbols, unsigned OptLevel,
269 bool Freestanding, const TargetMachineBuilder &TMBuilder) {
270 if (CachePath.empty())
273 if (!Index.modulePaths().count(ModuleID))
274 // The module does not have an entry, it can't have a hash at all
277 // Compute the unique hash for this entry
278 // This is based on the current compiler version, the module itself, the
279 // export list, the hash for every single module in the import list, the
280 // list of ResolvedODR for the module, and the list of preserved symbols.
282 // Include the hash for the current module
283 auto ModHash = Index.getModuleHash(ModuleID);
285 if (all_of(ModHash, [](uint32_t V) { return V == 0; }))
286 // No hash entry, no caching!
291 // Include the parts of the LTO configuration that affect code generation.
292 auto AddString = [&](StringRef Str) {
294 Hasher.update(ArrayRef<uint8_t>{0});
296 auto AddUnsigned = [&](unsigned I) {
302 Hasher.update(ArrayRef<uint8_t>{Data, 4});
305 // Start with the compiler revision
306 Hasher.update(LLVM_VERSION_STRING);
307 #ifdef HAVE_LLVM_REVISION
308 Hasher.update(LLVM_REVISION);
311 // Hash the optimization level and the target machine settings.
312 AddString(TMBuilder.MCpu);
313 // FIXME: Hash more of Options. For now all clients initialize Options from
314 // command-line flags (which is unsupported in production), but may set
315 // RelaxELFRelocations. The clang driver can also pass FunctionSections,
316 // DataSections and DebuggerTuning via command line flags.
317 AddUnsigned(TMBuilder.Options.RelaxELFRelocations);
318 AddUnsigned(TMBuilder.Options.FunctionSections);
319 AddUnsigned(TMBuilder.Options.DataSections);
320 AddUnsigned((unsigned)TMBuilder.Options.DebuggerTuning);
321 AddString(TMBuilder.MAttr);
322 if (TMBuilder.RelocModel)
323 AddUnsigned(*TMBuilder.RelocModel);
324 AddUnsigned(TMBuilder.CGOptLevel);
325 AddUnsigned(OptLevel);
326 AddUnsigned(Freestanding);
328 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
329 for (auto F : ExportList)
330 // The export list can impact the internalization, be conservative here
331 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
333 // Include the hash for every module we import functions from
334 for (auto &Entry : ImportList) {
335 auto ModHash = Index.getModuleHash(Entry.first());
336 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
339 // Include the hash for the resolved ODR.
340 for (auto &Entry : ResolvedODR) {
341 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
342 sizeof(GlobalValue::GUID)));
343 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
344 sizeof(GlobalValue::LinkageTypes)));
347 // Include the hash for the preserved symbols.
348 for (auto &Entry : PreservedSymbols) {
349 if (DefinedFunctions.count(Entry))
351 ArrayRef<uint8_t>((const uint8_t *)&Entry, sizeof(GlobalValue::GUID)));
354 // This choice of file name allows the cache to be pruned (see pruneCache()
355 // in include/llvm/Support/CachePruning.h).
356 sys::path::append(EntryPath, CachePath,
357 "llvmcache-" + toHex(Hasher.result()));
360 // Access the path to this entry in the cache.
361 StringRef getEntryPath() { return EntryPath; }
363 // Try loading the buffer for this cache entry.
364 ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
365 if (EntryPath.empty())
366 return std::error_code();
367 return MemoryBuffer::getFile(EntryPath);
370 // Cache the Produced object file
371 void write(const MemoryBuffer &OutputBuffer) {
372 if (EntryPath.empty())
375 // Write to a temporary to avoid race condition
376 SmallString<128> TempFilename;
379 sys::fs::createTemporaryFile("Thin", "tmp.o", TempFD, TempFilename);
381 errs() << "Error: " << EC.message() << "\n";
382 report_fatal_error("ThinLTO: Can't get a temporary file");
385 raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
386 OS << OutputBuffer.getBuffer();
388 // Rename to final destination (hopefully race condition won't matter here)
389 EC = sys::fs::rename(TempFilename, EntryPath);
391 sys::fs::remove(TempFilename);
392 raw_fd_ostream OS(EntryPath, EC, sys::fs::F_None);
394 report_fatal_error(Twine("Failed to open ") + EntryPath +
395 " to save cached entry\n");
396 OS << OutputBuffer.getBuffer();
401 static std::unique_ptr<MemoryBuffer>
402 ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
403 StringMap<MemoryBufferRef> &ModuleMap, TargetMachine &TM,
404 const FunctionImporter::ImportMapTy &ImportList,
405 const FunctionImporter::ExportSetTy &ExportList,
406 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
407 const GVSummaryMapTy &DefinedGlobals,
408 const ThinLTOCodeGenerator::CachingOptions &CacheOptions,
409 bool DisableCodeGen, StringRef SaveTempsDir,
410 bool Freestanding, unsigned OptLevel, unsigned count) {
412 // "Benchmark"-like optimization: single-source case
413 bool SingleModule = (ModuleMap.size() == 1);
416 promoteModule(TheModule, Index);
418 // Apply summary-based LinkOnce/Weak resolution decisions.
419 thinLTOResolveWeakForLinkerModule(TheModule, DefinedGlobals);
421 // Save temps: after promotion.
422 saveTempBitcode(TheModule, SaveTempsDir, count, ".1.promoted.bc");
425 // Be friendly and don't nuke totally the module when the client didn't
426 // supply anything to preserve.
427 if (!ExportList.empty() || !GUIDPreservedSymbols.empty()) {
428 // Apply summary-based internalization decisions.
429 thinLTOInternalizeModule(TheModule, DefinedGlobals);
432 // Save internalized bitcode
433 saveTempBitcode(TheModule, SaveTempsDir, count, ".2.internalized.bc");
436 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
438 // Save temps: after cross-module import.
439 saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
442 optimizeModule(TheModule, TM, OptLevel, Freestanding);
444 saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
446 if (DisableCodeGen) {
447 // Configured to stop before CodeGen, serialize the bitcode and return.
448 SmallVector<char, 128> OutputBuffer;
450 raw_svector_ostream OS(OutputBuffer);
451 ProfileSummaryInfo PSI(TheModule);
452 auto Index = buildModuleSummaryIndex(TheModule, nullptr, nullptr);
453 WriteBitcodeToFile(&TheModule, OS, true, &Index);
455 return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
458 return codegenModule(TheModule, TM);
461 /// Resolve LinkOnce/Weak symbols. Record resolutions in the \p ResolvedODR map
462 /// for caching, and in the \p Index for application during the ThinLTO
463 /// backends. This is needed for correctness for exported symbols (ensure
464 /// at least one copy kept) and a compile-time optimization (to drop duplicate
465 /// copies when possible).
466 static void resolveWeakForLinkerInIndex(
467 ModuleSummaryIndex &Index,
468 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
471 DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
472 computePrevailingCopies(Index, PrevailingCopy);
474 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
475 const auto &Prevailing = PrevailingCopy.find(GUID);
476 // Not in map means that there was only one copy, which must be prevailing.
477 if (Prevailing == PrevailingCopy.end())
479 return Prevailing->second == S;
482 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
483 GlobalValue::GUID GUID,
484 GlobalValue::LinkageTypes NewLinkage) {
485 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
488 thinLTOResolveWeakForLinkerInIndex(Index, isPrevailing, recordNewLinkage);
491 // Initialize the TargetMachine builder for a given Triple
492 static void initTMBuilder(TargetMachineBuilder &TMBuilder,
493 const Triple &TheTriple) {
494 // Set a default CPU for Darwin triples (copied from LTOCodeGenerator).
495 // FIXME this looks pretty terrible...
496 if (TMBuilder.MCpu.empty() && TheTriple.isOSDarwin()) {
497 if (TheTriple.getArch() == llvm::Triple::x86_64)
498 TMBuilder.MCpu = "core2";
499 else if (TheTriple.getArch() == llvm::Triple::x86)
500 TMBuilder.MCpu = "yonah";
501 else if (TheTriple.getArch() == llvm::Triple::aarch64)
502 TMBuilder.MCpu = "cyclone";
504 TMBuilder.TheTriple = std::move(TheTriple);
507 } // end anonymous namespace
509 void ThinLTOCodeGenerator::addModule(StringRef Identifier, StringRef Data) {
510 ThinLTOBuffer Buffer(Data, Identifier);
511 if (Modules.empty()) {
512 // First module added, so initialize the triple and some options
515 ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
516 Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
518 TripleStr = *TripleOrErr;
519 Triple TheTriple(TripleStr);
520 initTMBuilder(TMBuilder, Triple(TheTriple));
526 ErrorOr<std::string> TripleOrErr = expectedToErrorOrAndEmitErrors(
527 Context, getBitcodeTargetTriple(Buffer.getMemBuffer()));
529 TripleStr = *TripleOrErr;
530 assert(TMBuilder.TheTriple.str() == TripleStr &&
531 "ThinLTO modules with different triple not supported");
534 Modules.push_back(Buffer);
537 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
538 PreservedSymbols.insert(Name);
541 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
542 // FIXME: At the moment, we don't take advantage of this extra information,
543 // we're conservatively considering cross-references as preserved.
544 // CrossReferencedSymbols.insert(Name);
545 PreservedSymbols.insert(Name);
548 // TargetMachine factory
549 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
551 const Target *TheTarget =
552 TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
554 report_fatal_error("Can't load target for this Triple: " + ErrMsg);
557 // Use MAttr as the default set of features.
558 SubtargetFeatures Features(MAttr);
559 Features.getDefaultSubtargetFeatures(TheTriple);
560 std::string FeatureStr = Features.getString();
562 return std::unique_ptr<TargetMachine>(TheTarget->createTargetMachine(
563 TheTriple.str(), MCpu, FeatureStr, Options, RelocModel,
564 CodeModel::Default, CGOptLevel));
568 * Produce the combined summary index from all the bitcode files:
571 std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
572 std::unique_ptr<ModuleSummaryIndex> CombinedIndex;
573 uint64_t NextModuleId = 0;
574 for (auto &ModuleBuffer : Modules) {
575 Expected<std::unique_ptr<object::ModuleSummaryIndexObjectFile>> ObjOrErr =
576 object::ModuleSummaryIndexObjectFile::create(
577 ModuleBuffer.getMemBuffer());
580 logAllUnhandledErrors(
581 ObjOrErr.takeError(), errs(),
582 "error: can't create ModuleSummaryIndexObjectFile for buffer: ");
585 auto Index = (*ObjOrErr)->takeIndex();
587 CombinedIndex->mergeFrom(std::move(Index), ++NextModuleId);
589 CombinedIndex = std::move(Index);
592 return CombinedIndex;
596 * Perform promotion and renaming of exported internal functions.
597 * Index is updated to reflect linkage changes from weak resolution.
599 void ThinLTOCodeGenerator::promote(Module &TheModule,
600 ModuleSummaryIndex &Index) {
601 auto ModuleCount = Index.modulePaths().size();
602 auto ModuleIdentifier = TheModule.getModuleIdentifier();
604 // Collect for each module the list of function it defines (GUID -> Summary).
605 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries;
606 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
608 // Convert the preserved symbols set from string to GUID
609 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
610 PreservedSymbols, Triple(TheModule.getTargetTriple()));
612 // Compute "dead" symbols, we don't want to import/export these!
613 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
615 // Generate import/export list
616 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
617 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
618 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
619 ExportLists, &DeadSymbols);
621 // Resolve LinkOnce/Weak symbols.
622 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
623 resolveWeakForLinkerInIndex(Index, ResolvedODR);
625 thinLTOResolveWeakForLinkerModule(
626 TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
628 // Promote the exported values in the index, so that they are promoted
630 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
631 const auto &ExportList = ExportLists.find(ModuleIdentifier);
632 return (ExportList != ExportLists.end() &&
633 ExportList->second.count(GUID)) ||
634 GUIDPreservedSymbols.count(GUID);
636 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
638 promoteModule(TheModule, Index);
642 * Perform cross-module importing for the module identified by ModuleIdentifier.
644 void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
645 ModuleSummaryIndex &Index) {
646 auto ModuleMap = generateModuleMap(Modules);
647 auto ModuleCount = Index.modulePaths().size();
649 // Collect for each module the list of function it defines (GUID -> Summary).
650 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
651 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
653 // Convert the preserved symbols set from string to GUID
654 auto GUIDPreservedSymbols = computeGUIDPreservedSymbols(
655 PreservedSymbols, Triple(TheModule.getTargetTriple()));
657 // Compute "dead" symbols, we don't want to import/export these!
658 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
660 // Generate import/export list
661 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
662 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
663 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
664 ExportLists, &DeadSymbols);
665 auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
667 crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
671 * Compute the list of summaries needed for importing into module.
673 void ThinLTOCodeGenerator::gatherImportedSummariesForModule(
674 StringRef ModulePath, ModuleSummaryIndex &Index,
675 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
676 auto ModuleCount = Index.modulePaths().size();
678 // Collect for each module the list of function it defines (GUID -> Summary).
679 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
680 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
682 // Generate import/export list
683 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
684 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
685 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
688 llvm::gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
689 ImportLists[ModulePath],
690 ModuleToSummariesForIndex);
694 * Emit the list of files needed for importing into module.
696 void ThinLTOCodeGenerator::emitImports(StringRef ModulePath,
697 StringRef OutputName,
698 ModuleSummaryIndex &Index) {
699 auto ModuleCount = Index.modulePaths().size();
701 // Collect for each module the list of function it defines (GUID -> Summary).
702 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
703 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
705 // Generate import/export list
706 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
707 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
708 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
712 if ((EC = EmitImportsFiles(ModulePath, OutputName, ImportLists[ModulePath])))
713 report_fatal_error(Twine("Failed to open ") + OutputName +
714 " to save imports lists\n");
718 * Perform internalization. Index is updated to reflect linkage changes.
720 void ThinLTOCodeGenerator::internalize(Module &TheModule,
721 ModuleSummaryIndex &Index) {
722 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
723 auto ModuleCount = Index.modulePaths().size();
724 auto ModuleIdentifier = TheModule.getModuleIdentifier();
726 // Convert the preserved symbols set from string to GUID
727 auto GUIDPreservedSymbols =
728 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
730 // Collect for each module the list of function it defines (GUID -> Summary).
731 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
732 Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
734 // Compute "dead" symbols, we don't want to import/export these!
735 auto DeadSymbols = computeDeadSymbols(Index, GUIDPreservedSymbols);
737 // Generate import/export list
738 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
739 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
740 ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
741 ExportLists, &DeadSymbols);
742 auto &ExportList = ExportLists[ModuleIdentifier];
744 // Be friendly and don't nuke totally the module when the client didn't
745 // supply anything to preserve.
746 if (ExportList.empty() && GUIDPreservedSymbols.empty())
750 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
751 const auto &ExportList = ExportLists.find(ModuleIdentifier);
752 return (ExportList != ExportLists.end() &&
753 ExportList->second.count(GUID)) ||
754 GUIDPreservedSymbols.count(GUID);
756 thinLTOInternalizeAndPromoteInIndex(Index, isExported);
757 thinLTOInternalizeModule(TheModule,
758 ModuleToDefinedGVSummaries[ModuleIdentifier]);
762 * Perform post-importing ThinLTO optimizations.
764 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
765 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
768 optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding);
772 * Perform ThinLTO CodeGen.
774 std::unique_ptr<MemoryBuffer> ThinLTOCodeGenerator::codegen(Module &TheModule) {
775 initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
776 return codegenModule(TheModule, *TMBuilder.create());
779 /// Write out the generated object file, either from CacheEntryPath or from
780 /// OutputBuffer, preferring hard-link when possible.
781 /// Returns the path to the generated file in SavedObjectsDirectoryPath.
782 static std::string writeGeneratedObject(int count, StringRef CacheEntryPath,
783 StringRef SavedObjectsDirectoryPath,
784 const MemoryBuffer &OutputBuffer) {
785 SmallString<128> OutputPath(SavedObjectsDirectoryPath);
786 llvm::sys::path::append(OutputPath, Twine(count) + ".thinlto.o");
787 OutputPath.c_str(); // Ensure the string is null terminated.
788 if (sys::fs::exists(OutputPath))
789 sys::fs::remove(OutputPath);
791 // We don't return a memory buffer to the linker, just a list of files.
792 if (!CacheEntryPath.empty()) {
793 // Cache is enabled, hard-link the entry (or copy if hard-link fails).
794 auto Err = sys::fs::create_hard_link(CacheEntryPath, OutputPath);
796 return OutputPath.str();
797 // Hard linking failed, try to copy.
798 Err = sys::fs::copy_file(CacheEntryPath, OutputPath);
800 return OutputPath.str();
801 // Copy failed (could be because the CacheEntry was removed from the cache
802 // in the meantime by another process), fall back and try to write down the
803 // buffer to the output.
804 errs() << "error: can't link or copy from cached entry '" << CacheEntryPath
805 << "' to '" << OutputPath << "'\n";
807 // No cache entry, just write out the buffer.
809 raw_fd_ostream OS(OutputPath, Err, sys::fs::F_None);
811 report_fatal_error("Can't open output '" + OutputPath + "'\n");
812 OS << OutputBuffer.getBuffer();
813 return OutputPath.str();
816 // Main entry point for the ThinLTO processing
817 void ThinLTOCodeGenerator::run() {
818 // Prepare the resulting object vector
819 assert(ProducedBinaries.empty() && "The generator should not be reused");
820 if (SavedObjectsDirectoryPath.empty())
821 ProducedBinaries.resize(Modules.size());
823 sys::fs::create_directories(SavedObjectsDirectoryPath);
825 sys::fs::is_directory(SavedObjectsDirectoryPath, IsDir);
827 report_fatal_error("Unexistent dir: '" + SavedObjectsDirectoryPath + "'");
828 ProducedBinaryFiles.resize(Modules.size());
832 // Perform only parallel codegen and return.
835 for (auto &ModuleBuffer : Modules) {
836 Pool.async([&](int count) {
838 Context.setDiscardValueNames(LTODiscardValueNames);
842 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
843 /*IsImporting*/ false);
846 auto OutputBuffer = codegen(*TheModule);
847 if (SavedObjectsDirectoryPath.empty())
848 ProducedBinaries[count] = std::move(OutputBuffer);
850 ProducedBinaryFiles[count] = writeGeneratedObject(
851 count, "", SavedObjectsDirectoryPath, *OutputBuffer);
858 // Sequential linking phase
859 auto Index = linkCombinedIndex();
861 // Save temps: index.
862 if (!SaveTempsDir.empty()) {
863 auto SaveTempPath = SaveTempsDir + "index.bc";
865 raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
867 report_fatal_error(Twine("Failed to open ") + SaveTempPath +
868 " to save optimized bitcode\n");
869 WriteIndexToFile(*Index, OS);
873 // Prepare the module map.
874 auto ModuleMap = generateModuleMap(Modules);
875 auto ModuleCount = Modules.size();
877 // Collect for each module the list of function it defines (GUID -> Summary).
878 StringMap<GVSummaryMapTy> ModuleToDefinedGVSummaries(ModuleCount);
879 Index->collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
881 // Convert the preserved symbols set from string to GUID, this is needed for
882 // computing the caching hash and the internalization.
883 auto GUIDPreservedSymbols =
884 computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
886 // Compute "dead" symbols, we don't want to import/export these!
887 auto DeadSymbols = computeDeadSymbols(*Index, GUIDPreservedSymbols);
889 // Collect the import/export lists for all modules from the call-graph in the
891 StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
892 StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
893 ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
894 ExportLists, &DeadSymbols);
896 // We use a std::map here to be able to have a defined ordering when
897 // producing a hash for the cache entry.
898 // FIXME: we should be able to compute the caching hash for the entry based
899 // on the index, and nuke this map.
900 StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
902 // Resolve LinkOnce/Weak symbols, this has to be computed early because it
903 // impacts the caching.
904 resolveWeakForLinkerInIndex(*Index, ResolvedODR);
906 auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
907 const auto &ExportList = ExportLists.find(ModuleIdentifier);
908 return (ExportList != ExportLists.end() &&
909 ExportList->second.count(GUID)) ||
910 GUIDPreservedSymbols.count(GUID);
913 // Use global summary-based analysis to identify symbols that can be
914 // internalized (because they aren't exported or preserved as per callback).
915 // Changes are made in the index, consumed in the ThinLTO backends.
916 thinLTOInternalizeAndPromoteInIndex(*Index, isExported);
918 // Make sure that every module has an entry in the ExportLists and
919 // ResolvedODR maps to enable threaded access to these maps below.
920 for (auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
921 ExportLists[DefinedGVSummaries.first()];
922 ResolvedODR[DefinedGVSummaries.first()];
925 // Compute the ordering we will process the inputs: the rough heuristic here
926 // is to sort them per size so that the largest module get schedule as soon as
927 // possible. This is purely a compile-time optimization.
928 std::vector<int> ModulesOrdering;
929 ModulesOrdering.resize(Modules.size());
930 std::iota(ModulesOrdering.begin(), ModulesOrdering.end(), 0);
931 std::sort(ModulesOrdering.begin(), ModulesOrdering.end(),
932 [&](int LeftIndex, int RightIndex) {
933 auto LSize = Modules[LeftIndex].getBuffer().size();
934 auto RSize = Modules[RightIndex].getBuffer().size();
935 return LSize > RSize;
938 // Parallel optimizer + codegen
940 ThreadPool Pool(ThreadCount);
941 for (auto IndexCount : ModulesOrdering) {
942 auto &ModuleBuffer = Modules[IndexCount];
943 Pool.async([&](int count) {
944 auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier();
945 auto &ExportList = ExportLists[ModuleIdentifier];
947 auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier];
949 // The module may be cached, this helps handling it.
950 ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
951 ImportLists[ModuleIdentifier], ExportList,
952 ResolvedODR[ModuleIdentifier],
953 DefinedFunctions, GUIDPreservedSymbols,
954 OptLevel, Freestanding, TMBuilder);
955 auto CacheEntryPath = CacheEntry.getEntryPath();
958 auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
959 DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss") << " '"
960 << CacheEntryPath << "' for buffer " << count << " "
961 << ModuleIdentifier << "\n");
965 if (SavedObjectsDirectoryPath.empty())
966 ProducedBinaries[count] = std::move(ErrOrBuffer.get());
968 ProducedBinaryFiles[count] = writeGeneratedObject(
969 count, CacheEntryPath, SavedObjectsDirectoryPath,
976 Context.setDiscardValueNames(LTODiscardValueNames);
977 Context.enableDebugTypeODRUniquing();
978 auto DiagFileOrErr = lto::setupOptimizationRemarks(
979 Context, LTORemarksFilename, LTOPassRemarksWithHotness, count);
980 if (!DiagFileOrErr) {
981 errs() << "Error: " << toString(DiagFileOrErr.takeError()) << "\n";
982 report_fatal_error("ThinLTO: Can't get an output file for the "
988 loadModuleFromBuffer(ModuleBuffer.getMemBuffer(), Context, false,
989 /*IsImporting*/ false);
991 // Save temps: original file.
992 saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
994 auto &ImportList = ImportLists[ModuleIdentifier];
995 // Run the main process now, and generates a binary
996 auto OutputBuffer = ProcessThinLTOModule(
997 *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
998 ExportList, GUIDPreservedSymbols,
999 ModuleToDefinedGVSummaries[ModuleIdentifier], CacheOptions,
1000 DisableCodeGen, SaveTempsDir, Freestanding, OptLevel, count);
1002 // Commit to the cache (if enabled)
1003 CacheEntry.write(*OutputBuffer);
1005 if (SavedObjectsDirectoryPath.empty()) {
1006 // We need to generated a memory buffer for the linker.
1007 if (!CacheEntryPath.empty()) {
1008 // Cache is enabled, reload from the cache
1009 // We do this to lower memory pressuree: the buffer is on the heap
1010 // and releasing it frees memory that can be used for the next input
1011 // file. The final binary link will read from the VFS cache
1012 // (hopefully!) or from disk if the memory pressure wasn't too high.
1013 auto ReloadedBufferOrErr = CacheEntry.tryLoadingBuffer();
1014 if (auto EC = ReloadedBufferOrErr.getError()) {
1015 // On error, keeping the preexisting buffer and printing a
1016 // diagnostic is more friendly than just crashing.
1017 errs() << "error: can't reload cached file '" << CacheEntryPath
1018 << "': " << EC.message() << "\n";
1020 OutputBuffer = std::move(*ReloadedBufferOrErr);
1023 ProducedBinaries[count] = std::move(OutputBuffer);
1026 ProducedBinaryFiles[count] = writeGeneratedObject(
1027 count, CacheEntryPath, SavedObjectsDirectoryPath, *OutputBuffer);
1032 pruneCache(CacheOptions.Path, CacheOptions.Policy);
1034 // If statistics were requested, print them out now.
1035 if (llvm::AreStatisticsEnabled())
1036 llvm::PrintStatistics();