OSDN Git Service

Generate SHA-1 build ID for host-generated *.oat files (1/2).
authorAlexey Alexandrov <aalexand@google.com>
Mon, 19 Sep 2016 16:33:49 +0000 (09:33 -0700)
committerAlexey Alexandrov <aalexand@google.com>
Thu, 6 Oct 2016 02:20:14 +0000 (19:20 -0700)
For host-generated *.oat files, generate a SHA-1 build ID based on the
file content and write it to .note.gnu.build-id ELF section.  This
should allow various developer tools like profilers correlate the data
captured for files like boot.oat on the device with the corresponding
known version of the file during an offline analysis.

Test: Verified that boot.oat contains the build ID section now (with
      this change and https://android-review.googlesource.com/#/c/275630
      applied)
Test: Added ElfWriterTest::CheckBuildIdPresent test.
Test: make test-art-host
Bug: 31292208
Change-Id: Ie5e89da2ef87e34c27c0237ab34ddc7d2dc0aa3b

build/Android.oat.mk
compiler/Android.bp
compiler/driver/compiler_options.cc
compiler/driver/compiler_options.h
compiler/elf_builder.h
compiler/elf_writer_quick.cc
compiler/elf_writer_test.cc
dex2oat/dex2oat.cc

index c4887e6..e297b4f 100644 (file)
@@ -122,7 +122,8 @@ $$(core_image_name): $$(HOST_CORE_DEX_LOCATIONS) $$(core_dex2oat_dependency)
          --oat-location=$$(PRIVATE_CORE_OAT_NAME) --image=$$(PRIVATE_CORE_IMG_NAME) \
          --base=$$(LIBART_IMG_HOST_BASE_ADDRESS) --instruction-set=$$($(3)ART_HOST_ARCH) \
          $$(LOCAL_$(3)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES_OPTION) \
-         --host --android-root=$$(HOST_OUT) --include-patch-information --generate-debug-info \
+         --host --android-root=$$(HOST_OUT) --include-patch-information \
+         --generate-debug-info --generate-build-id \
          $$(PRIVATE_CORE_MULTI_PARAM) $$(PRIVATE_CORE_COMPILE_OPTIONS)
 
 $$(core_oat_name): $$(core_image_name)
@@ -239,7 +240,8 @@ $$(core_image_name): $$(TARGET_CORE_DEX_FILES) $$(core_dex2oat_dependency)
          --base=$$(LIBART_IMG_TARGET_BASE_ADDRESS) --instruction-set=$$($(3)TARGET_ARCH) \
          --instruction-set-variant=$$($(3)DEX2OAT_TARGET_CPU_VARIANT) \
          --instruction-set-features=$$($(3)DEX2OAT_TARGET_INSTRUCTION_SET_FEATURES) \
-         --android-root=$$(PRODUCT_OUT)/system --include-patch-information --generate-debug-info \
+         --android-root=$$(PRODUCT_OUT)/system --include-patch-information \
+         --generate-debug-info --generate-build-id \
          $$(PRIVATE_CORE_COMPILE_OPTIONS) || (rm $$(PRIVATE_CORE_OAT_NAME); exit 1)
 
 $$(core_oat_name): $$(core_image_name)
index 2556178..d9ed067 100644 (file)
@@ -190,6 +190,14 @@ art_cc_defaults {
     ],
     include_dirs: ["art/disassembler"],
     export_include_dirs: ["."],
+
+    // For SHA-1 checksumming of build ID
+    static: {
+        whole_static_libs: ["libcrypto"],
+    },
+    shared: {
+        shared_libs: ["libcrypto"],
+    },
 }
 
 gensrcs {
index cbcc169..c222f90 100644 (file)
@@ -37,6 +37,7 @@ CompilerOptions::CompilerOptions()
       debuggable_(false),
       generate_debug_info_(kDefaultGenerateDebugInfo),
       generate_mini_debug_info_(kDefaultGenerateMiniDebugInfo),
+      generate_build_id_(false),
       implicit_null_checks_(true),
       implicit_so_checks_(true),
       implicit_suspend_checks_(false),
@@ -97,6 +98,7 @@ CompilerOptions::CompilerOptions(CompilerFilter::Filter compiler_filter,
       debuggable_(debuggable),
       generate_debug_info_(generate_debug_info),
       generate_mini_debug_info_(kDefaultGenerateMiniDebugInfo),
+      generate_build_id_(false),
       implicit_null_checks_(implicit_null_checks),
       implicit_so_checks_(implicit_so_checks),
       implicit_suspend_checks_(implicit_suspend_checks),
@@ -196,6 +198,10 @@ bool CompilerOptions::ParseCompilerOption(const StringPiece& option, UsageFn Usa
     generate_mini_debug_info_ = true;
   } else if (option == "--no-generate-mini-debug-info") {
     generate_mini_debug_info_ = false;
+  } else if (option == "--generate-build-id") {
+    generate_build_id_ = true;
+  } else if (option == "--no-generate-build-id") {
+    generate_build_id_ = false;
   } else if (option == "--debuggable") {
     debuggable_ = true;
   } else if (option.starts_with("--top-k-profile-threshold=")) {
index 8e4a775..3c920d9 100644 (file)
@@ -187,6 +187,10 @@ class CompilerOptions FINAL {
     return generate_mini_debug_info_;
   }
 
+  bool GetGenerateBuildId() const {
+    return generate_build_id_;
+  }
+
   bool GetImplicitNullChecks() const {
     return implicit_null_checks_;
   }
@@ -297,6 +301,7 @@ class CompilerOptions FINAL {
   bool debuggable_;
   bool generate_debug_info_;
   bool generate_mini_debug_info_;
+  bool generate_build_id_;
   bool implicit_null_checks_;
   bool implicit_so_checks_;
   bool implicit_suspend_checks_;
index 73240be..31a7529 100644 (file)
@@ -36,6 +36,7 @@ namespace art {
 // The basic layout of the elf file:
 //   Elf_Ehdr                    - The ELF header.
 //   Elf_Phdr[]                  - Program headers for the linker.
+//   .note.gnu.build-id          - Optional build ID section (SHA-1 digest).
 //   .rodata                     - DEX files and oat metadata.
 //   .text                       - Compiled code.
 //   .bss                        - Zero-initialized writeable section.
@@ -75,6 +76,10 @@ template <typename ElfTypes>
 class ElfBuilder FINAL {
  public:
   static constexpr size_t kMaxProgramHeaders = 16;
+  // SHA-1 digest.  Not using SHA_DIGEST_LENGTH from openssl/sha.h to avoid
+  // spreading this header dependency for just this single constant.
+  static constexpr size_t kBuildIdLen = 20;
+
   using Elf_Addr = typename ElfTypes::Addr;
   using Elf_Off = typename ElfTypes::Off;
   using Elf_Word = typename ElfTypes::Word;
@@ -458,6 +463,49 @@ class ElfBuilder FINAL {
     } abiflags_;
   };
 
+  class BuildIdSection FINAL : public Section {
+   public:
+    BuildIdSection(ElfBuilder<ElfTypes>* owner,
+                   const std::string& name,
+                   Elf_Word type,
+                   Elf_Word flags,
+                   const Section* link,
+                   Elf_Word info,
+                   Elf_Word align,
+                   Elf_Word entsize)
+        : Section(owner, name, type, flags, link, info, align, entsize),
+          digest_start_(-1) {
+    }
+
+    void Write() {
+      // The size fields are 32-bit on both 32-bit and 64-bit systems, confirmed
+      // with the 64-bit linker and libbfd code. The size of name and desc must
+      // be a multiple of 4 and it currently is.
+      this->WriteUint32(4);  // namesz.
+      this->WriteUint32(kBuildIdLen);  // descsz.
+      this->WriteUint32(3);  // type = NT_GNU_BUILD_ID.
+      this->WriteFully("GNU", 4);  // name.
+      digest_start_ = this->Seek(0, kSeekCurrent);
+      static_assert(kBuildIdLen % 4 == 0, "expecting a mutliple of 4 for build ID length");
+      this->WriteFully(std::string(kBuildIdLen, '\0').c_str(), kBuildIdLen);  // desc.
+    }
+
+    off_t GetDigestStart() {
+      CHECK_GT(digest_start_, 0);
+      return digest_start_;
+    }
+
+   private:
+    bool WriteUint32(uint32_t v) {
+      return this->WriteFully(&v, sizeof(v));
+    }
+
+    // File offset where the build ID digest starts.
+    // Populated with zeros first, then updated with the actual value as the
+    // very last thing in the output file creation.
+    off_t digest_start_;
+  };
+
   ElfBuilder(InstructionSet isa, const InstructionSetFeatures* features, OutputStream* output)
       : isa_(isa),
         features_(features),
@@ -479,6 +527,7 @@ class ElfBuilder FINAL {
         shstrtab_(this, ".shstrtab", 0, 1),
         abiflags_(this, ".MIPS.abiflags", SHT_MIPS_ABIFLAGS, SHF_ALLOC, nullptr, 0, kPageSize, 0,
                   isa, features),
+        build_id_(this, ".note.gnu.build-id", SHT_NOTE, SHF_ALLOC, nullptr, 0, 4, 0),
         started_(false),
         write_program_headers_(false),
         loaded_size_(0u),
@@ -489,6 +538,7 @@ class ElfBuilder FINAL {
     dynamic_.phdr_type_ = PT_DYNAMIC;
     eh_frame_hdr_.phdr_type_ = PT_GNU_EH_FRAME;
     abiflags_.phdr_type_ = PT_MIPS_ABIFLAGS;
+    build_id_.phdr_type_ = PT_NOTE;
   }
   ~ElfBuilder() {}
 
@@ -741,6 +791,17 @@ class ElfBuilder FINAL {
     abiflags_.End();
   }
 
+  void WriteBuildIdSection() {
+    build_id_.Start();
+    build_id_.Write();
+    build_id_.End();
+  }
+
+  void WriteBuildId(uint8_t build_id[kBuildIdLen]) {
+    stream_.Seek(build_id_.GetDigestStart(), kSeekSet);
+    stream_.WriteFully(build_id, kBuildIdLen);
+  }
+
   // Returns true if all writes and seeks on the output stream succeeded.
   bool Good() {
     return stream_.Good();
@@ -932,6 +993,7 @@ class ElfBuilder FINAL {
   Section debug_line_;
   StringSection shstrtab_;
   AbiflagsSection abiflags_;
+  BuildIdSection build_id_;
   std::vector<std::unique_ptr<Section>> other_sections_;
 
   // List of used section in the order in which they were written.
index 36cd232..0d6575c 100644 (file)
@@ -16,6 +16,7 @@
 
 #include "elf_writer_quick.h"
 
+#include <openssl/sha.h>
 #include <unordered_map>
 #include <unordered_set>
 
@@ -126,6 +127,8 @@ class ElfWriterQuick FINAL : public ElfWriter {
   std::unique_ptr<DebugInfoTask> debug_info_task_;
   std::unique_ptr<ThreadPool> debug_info_thread_pool_;
 
+  void ComputeFileBuildId(uint8_t (*build_id)[ElfBuilder<ElfTypes>::kBuildIdLen]);
+
   DISALLOW_IMPLICIT_CONSTRUCTORS(ElfWriterQuick);
 };
 
@@ -167,6 +170,9 @@ ElfWriterQuick<ElfTypes>::~ElfWriterQuick() {}
 template <typename ElfTypes>
 void ElfWriterQuick<ElfTypes>::Start() {
   builder_->Start();
+  if (compiler_options_->GetGenerateBuildId()) {
+    builder_->WriteBuildIdSection();
+  }
 }
 
 template <typename ElfTypes>
@@ -275,11 +281,36 @@ void ElfWriterQuick<ElfTypes>::WritePatchLocations(
 template <typename ElfTypes>
 bool ElfWriterQuick<ElfTypes>::End() {
   builder_->End();
-
+  if (compiler_options_->GetGenerateBuildId()) {
+    uint8_t build_id[ElfBuilder<ElfTypes>::kBuildIdLen];
+    ComputeFileBuildId(&build_id);
+    builder_->WriteBuildId(build_id);
+  }
   return builder_->Good();
 }
 
 template <typename ElfTypes>
+void ElfWriterQuick<ElfTypes>::ComputeFileBuildId(
+    uint8_t (*build_id)[ElfBuilder<ElfTypes>::kBuildIdLen]) {
+  constexpr int kBufSize = 8192;
+  std::vector<char> buffer(kBufSize);
+  int64_t offset = 0;
+  SHA_CTX ctx;
+  SHA1_Init(&ctx);
+  while (true) {
+    int64_t bytes_read = elf_file_->Read(buffer.data(), kBufSize, offset);
+    CHECK_GE(bytes_read, 0);
+    if (bytes_read == 0) {
+      // End of file.
+      break;
+    }
+    SHA1_Update(&ctx, buffer.data(), bytes_read);
+    offset += bytes_read;
+  }
+  SHA1_Final(*build_id, &ctx);
+}
+
+template <typename ElfTypes>
 OutputStream* ElfWriterQuick<ElfTypes>::GetStream() {
   return builder_->GetStream();
 }
index d5f1663..b580049 100644 (file)
@@ -101,6 +101,25 @@ TEST_F(ElfWriterTest, dlsym) {
   }
 }
 
+TEST_F(ElfWriterTest, CheckBuildIdPresent) {
+  std::string elf_location = GetCoreOatLocation();
+  std::string elf_filename = GetSystemImageFilename(elf_location.c_str(), kRuntimeISA);
+  LOG(INFO) << "elf_filename=" << elf_filename;
+
+  std::unique_ptr<File> file(OS::OpenFileForReading(elf_filename.c_str()));
+  ASSERT_TRUE(file.get() != nullptr);
+  {
+    std::string error_msg;
+    std::unique_ptr<ElfFile> ef(ElfFile::Open(file.get(),
+                                              false,
+                                              false,
+                                              /*low_4gb*/false,
+                                              &error_msg));
+    CHECK(ef.get() != nullptr) << error_msg;
+    EXPECT_TRUE(ef->HasSection(".note.gnu.build-id"));
+  }
+}
+
 TEST_F(ElfWriterTest, EncodeDecodeOatPatches) {
   const std::vector<std::vector<uintptr_t>> test_data {
       { 0, 4, 8, 15, 128, 200 },
index 292aff4..3e8adc0 100644 (file)
@@ -339,6 +339,11 @@ NO_RETURN static void Usage(const char* fmt, ...) {
   UsageError("");
   UsageError("  --no-generate-mini-debug-info: Do not generate backtrace info.");
   UsageError("");
+  UsageError("  --generate-build-id: Generate GNU-compatible linker build ID ELF section with");
+  UsageError("      SHA-1 of the file content (and thus stable across identical builds)");
+  UsageError("");
+  UsageError("  --no-generate-build-id: Do not generate the build ID ELF section.");
+  UsageError("");
   UsageError("  --debuggable: Produce code debuggable with Java debugger.");
   UsageError("");
   UsageError("  --runtime-arg <argument>: used to specify various arguments for the runtime,");