From 608f2ce4b9870354079b9d63d40363914889f01a Mon Sep 17 00:00:00 2001 From: Jeff Hao Date: Wed, 19 Oct 2016 11:17:11 -0700 Subject: [PATCH] Add new --layout-profile compiler-filter for dex2oat. New compiler filter makes dex2oat call into dexlayout. Added basic test for --layout-profile filter to make sure dex2oat runs to completion and file is valid. Contests of file are not checked. Test: mm test-art-host-gtest-dexlayout_test Bug: 29921113 Change-Id: I4bd0dea3d3f1284c155d1d9dea80a48062e67770 --- compiler/Android.bp | 10 ++++- compiler/image_test.cc | 4 +- compiler/oat_test.cc | 12 ++++-- compiler/oat_writer.cc | 45 ++++++++++++++++++++-- compiler/oat_writer.h | 7 +++- dex2oat/Android.bp | 2 + dex2oat/dex2oat.cc | 5 ++- dex2oat/dex2oat_test.cc | 90 ++++++++++++++++++++++++++++++++++++++++++++ runtime/compiler_filter.cc | 8 ++++ runtime/compiler_filter.h | 1 + runtime/dex_file_verifier.cc | 2 +- 11 files changed, 174 insertions(+), 12 deletions(-) diff --git a/compiler/Android.bp b/compiler/Android.bp index e2a450d17..17373767c 100644 --- a/compiler/Android.bp +++ b/compiler/Android.bp @@ -254,7 +254,10 @@ art_cc_library { }, }, }, - shared_libs: ["libart"], + shared_libs: [ + "libart", + "libart-dexlayout", + ], } art_cc_library { @@ -291,7 +294,10 @@ art_cc_library { }, }, }, - shared_libs: ["libartd"], + shared_libs: [ + "libartd", + "libartd-dexlayout" + ], } art_cc_library { diff --git a/compiler/image_test.cc b/compiler/image_test.cc index fcb897945..5629dffce 100644 --- a/compiler/image_test.cc +++ b/compiler/image_test.cc @@ -211,7 +211,9 @@ void CompilationHelper::Compile(CompilerDriver* driver, &driver->GetCompilerOptions(), oat_file.GetFile())); elf_writers.back()->Start(); - oat_writers.emplace_back(new OatWriter(/*compiling_boot_image*/true, &timings)); + oat_writers.emplace_back(new OatWriter(/*compiling_boot_image*/true, + &timings, + /*profile_compilation_info*/nullptr)); } std::vector rodata; diff --git a/compiler/oat_test.cc b/compiler/oat_test.cc index 102637f01..94585769b 100644 --- a/compiler/oat_test.cc +++ b/compiler/oat_test.cc @@ -125,7 +125,9 @@ class OatTest : public CommonCompilerTest { SafeMap& key_value_store, bool verify) { TimingLogger timings("WriteElf", false, false); - OatWriter oat_writer(/*compiling_boot_image*/false, &timings); + OatWriter oat_writer(/*compiling_boot_image*/false, + &timings, + /*profile_compilation_info*/nullptr); for (const DexFile* dex_file : dex_files) { ArrayRef raw_dex_file( reinterpret_cast(&dex_file->GetHeader()), @@ -145,7 +147,9 @@ class OatTest : public CommonCompilerTest { SafeMap& key_value_store, bool verify) { TimingLogger timings("WriteElf", false, false); - OatWriter oat_writer(/*compiling_boot_image*/false, &timings); + OatWriter oat_writer(/*compiling_boot_image*/false, + &timings, + /*profile_compilation_info*/nullptr); for (const char* dex_filename : dex_filenames) { if (!oat_writer.AddDexFileSource(dex_filename, dex_filename)) { return false; @@ -161,7 +165,9 @@ class OatTest : public CommonCompilerTest { SafeMap& key_value_store, bool verify) { TimingLogger timings("WriteElf", false, false); - OatWriter oat_writer(/*compiling_boot_image*/false, &timings); + OatWriter oat_writer(/*compiling_boot_image*/false, + &timings, + /*profile_compilation_info*/nullptr); if (!oat_writer.AddZippedDexFilesSource(std::move(zip_fd), location)) { return false; } diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index bde00cf88..eed9d117a 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -33,6 +33,7 @@ #include "debug/method_debug_info.h" #include "dex/verification_results.h" #include "dex_file-inl.h" +#include "dexlayout.h" #include "driver/compiler_driver.h" #include "driver/compiler_options.h" #include "gc/space/image_space.h" @@ -276,7 +277,7 @@ class OatWriter::OatDexFile { DCHECK_EQ(static_cast(file_offset + offset_), out->Seek(0, kSeekCurrent)) \ << "file_offset=" << file_offset << " offset_=" << offset_ -OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings) +OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCompilationInfo* info) : write_state_(WriteState::kAddingDexFileSources), timings_(timings), raw_dex_files_(), @@ -337,7 +338,8 @@ OatWriter::OatWriter(bool compiling_boot_image, TimingLogger* timings) size_oat_class_method_bitmaps_(0), size_oat_class_method_offsets_(0), relative_patcher_(nullptr), - absolute_patch_locations_() { + absolute_patch_locations_(), + profile_compilation_info_(info) { } bool OatWriter::AddDexFileSource(const char* filename, @@ -2081,7 +2083,11 @@ bool OatWriter::WriteDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_ if (!SeekToDexFile(out, file, oat_dex_file)) { return false; } - if (oat_dex_file->source_.IsZipEntry()) { + if (profile_compilation_info_ != nullptr) { + if (!LayoutAndWriteDexFile(out, oat_dex_file)) { + return false; + } + } else if (oat_dex_file->source_.IsZipEntry()) { if (!WriteDexFile(out, file, oat_dex_file, oat_dex_file->source_.GetZipEntry())) { return false; } @@ -2146,6 +2152,39 @@ bool OatWriter::SeekToDexFile(OutputStream* out, File* file, OatDexFile* oat_dex return true; } +bool OatWriter::LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_file) { + TimingLogger::ScopedTiming split("Dex Layout", timings_); + std::string error_msg; + std::string location(oat_dex_file->GetLocation()); + std::unique_ptr dex_file; + if (oat_dex_file->source_.IsZipEntry()) { + ZipEntry* zip_entry = oat_dex_file->source_.GetZipEntry(); + std::unique_ptr mem_map( + zip_entry->ExtractToMemMap(location.c_str(), "classes.dex", &error_msg)); + dex_file = DexFile::Open(location, + zip_entry->GetCrc32(), + std::move(mem_map), + /* verify */ true, + /* verify_checksum */ true, + &error_msg); + } else { + DCHECK(oat_dex_file->source_.IsRawFile()); + File* raw_file = oat_dex_file->source_.GetRawFile(); + dex_file = DexFile::OpenDex(raw_file->Fd(), location, /* verify_checksum */ true, &error_msg); + } + Options options; + options.output_to_memmap_ = true; + DexLayout dex_layout(options, profile_compilation_info_, nullptr); + dex_layout.ProcessDexFile(location.c_str(), dex_file.get(), 0); + std::unique_ptr mem_map(dex_layout.GetAndReleaseMemMap()); + if (!WriteDexFile(out, oat_dex_file, mem_map->Begin())) { + return false; + } + // Set the checksum of the new oat dex file to be the original file's checksum. + oat_dex_file->dex_file_location_checksum_ = dex_file->GetLocationChecksum(); + return true; +} + bool OatWriter::WriteDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file, diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index b92ba769a..f9671d7c5 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -38,6 +38,7 @@ class BitVector; class CompiledMethod; class CompilerDriver; class ImageWriter; +class ProfileCompilationInfo; class OutputStream; class TimingLogger; class TypeLookupTable; @@ -110,7 +111,7 @@ class OatWriter { kDefault = kCreate }; - OatWriter(bool compiling_boot_image, TimingLogger* timings); + OatWriter(bool compiling_boot_image, TimingLogger* timings, ProfileCompilationInfo* info); // To produce a valid oat file, the user must first add sources with any combination of // - AddDexFileSource(), @@ -258,6 +259,7 @@ class OatWriter { bool WriteDexFiles(OutputStream* out, File* file); bool WriteDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file); bool SeekToDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file); + bool LayoutAndWriteDexFile(OutputStream* out, OatDexFile* oat_dex_file); bool WriteDexFile(OutputStream* out, File* file, OatDexFile* oat_dex_file, @@ -422,6 +424,9 @@ class OatWriter { // The locations of absolute patches relative to the start of the executable section. dchecked_vector absolute_patch_locations_; + // Profile info used to generate new layout of files. + ProfileCompilationInfo* profile_compilation_info_; + DISALLOW_COPY_AND_ASSIGN(OatWriter); }; diff --git a/dex2oat/Android.bp b/dex2oat/Android.bp index 05a5d0fa2..0924aec7f 100644 --- a/dex2oat/Android.bp +++ b/dex2oat/Android.bp @@ -89,6 +89,7 @@ art_cc_binary { ], static_libs: [ "libart-compiler", + "libart-dexlayout", "libart", "libvixl-arm", "libvixl-arm64", @@ -118,6 +119,7 @@ art_cc_binary { ], static_libs: [ "libartd-compiler", + "libartd-dexlayout", "libartd", "libvixld-arm", "libvixld-arm64", diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 65703a23f..e52e502ec 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -270,6 +270,7 @@ NO_RETURN static void Usage(const char* fmt, ...) { "|balanced" "|speed-profile" "|speed" + "|layout-profile" "|everything-profile" "|everything):"); UsageError(" select compiler filter."); @@ -2273,7 +2274,9 @@ class Dex2Oat FINAL { compiler_options_.get(), oat_file.get())); elf_writers_.back()->Start(); - oat_writers_.emplace_back(new OatWriter(IsBootImage(), timings_)); + bool do_dexlayout = compiler_options_->GetCompilerFilter() == CompilerFilter::kLayoutProfile; + oat_writers_.emplace_back(new OatWriter( + IsBootImage(), timings_, do_dexlayout ? profile_compilation_info_.get() : nullptr)); } } diff --git a/dex2oat/dex2oat_test.cc b/dex2oat/dex2oat_test.cc index fa3217879..2f34019d0 100644 --- a/dex2oat/dex2oat_test.cc +++ b/dex2oat/dex2oat_test.cc @@ -24,6 +24,7 @@ #include "base/logging.h" #include "base/macros.h" #include "base/stringprintf.h" +#include "dex_file-inl.h" #include "dex2oat_environment_test.h" #include "oat.h" #include "oat_file.h" @@ -551,4 +552,93 @@ TEST_F(Dex2oatVeryLargeTest, UseVeryLarge) { RunTest(CompilerFilter::kSpeed, true, { "--very-large-app-threshold=100" }); } +static const char kDexFileLayoutInputProfile[] = + "cHJvADAwMgABAAsAAAABAPUpbf5jbGFzc2VzLmRleAEA"; + +static void WriteFileBase64(const char* base64, const char* location) { + // Decode base64. + CHECK(base64 != nullptr); + size_t length; + std::unique_ptr bytes(DecodeBase64(base64, &length)); + CHECK(bytes.get() != nullptr); + + // Write to provided file. + std::unique_ptr file(OS::CreateEmptyFile(location)); + CHECK(file.get() != nullptr); + if (!file->WriteFully(bytes.get(), length)) { + PLOG(FATAL) << "Failed to write base64 as file"; + } + if (file->FlushCloseOrErase() != 0) { + PLOG(FATAL) << "Could not flush and close test file."; + } +} + +class Dex2oatLayoutTest : public Dex2oatTest { + protected: + void CheckFilter(CompilerFilter::Filter input ATTRIBUTE_UNUSED, + CompilerFilter::Filter result ATTRIBUTE_UNUSED) OVERRIDE { + // Ignore, we'll do our own checks. + } + + void RunTest() { + std::string dex_location = GetScratchDir() + "/DexNoOat.jar"; + std::string profile_location = GetScratchDir() + "/primary.prof"; + std::string odex_location = GetOdexDir() + "/DexOdexNoOat.odex"; + + Copy(GetDexSrc2(), dex_location); + WriteFileBase64(kDexFileLayoutInputProfile, profile_location.c_str()); + + const std::vector& extra_args = { "--profile-file=" + profile_location }; + GenerateOdexForTest(dex_location, odex_location, CompilerFilter::kLayoutProfile, extra_args); + + CheckValidity(); + ASSERT_TRUE(success_); + CheckResult(dex_location, odex_location); + } + void CheckResult(const std::string& dex_location, const std::string& odex_location) { + // Host/target independent checks. + std::string error_msg; + std::unique_ptr odex_file(OatFile::Open(odex_location.c_str(), + odex_location.c_str(), + nullptr, + nullptr, + false, + /*low_4gb*/false, + dex_location.c_str(), + &error_msg)); + ASSERT_TRUE(odex_file.get() != nullptr) << error_msg; + + for (const OatDexFile* oat_dex_file : odex_file->GetOatDexFiles()) { + std::unique_ptr dex_file = oat_dex_file->OpenDexFile(&error_msg); + ASSERT_TRUE(dex_file != nullptr); + uint32_t class_def_count = dex_file->NumClassDefs(); + ASSERT_LT(class_def_count, std::numeric_limits::max()); + } + + EXPECT_EQ(odex_file->GetCompilerFilter(), CompilerFilter::kLayoutProfile); + } + + // Check whether the dex2oat run was really successful. + void CheckValidity() { + if (kIsTargetBuild) { + CheckTargetValidity(); + } else { + CheckHostValidity(); + } + } + + void CheckTargetValidity() { + // TODO: Ignore for now. + } + + // On the host, we can get the dex2oat output. Here, look for "dex2oat took." + void CheckHostValidity() { + EXPECT_NE(output_.find("dex2oat took"), std::string::npos) << output_; + } + }; + +TEST_F(Dex2oatLayoutTest, TestLayout) { + RunTest(); +} + } // namespace art diff --git a/runtime/compiler_filter.cc b/runtime/compiler_filter.cc index dc197c106..6e3e1d87e 100644 --- a/runtime/compiler_filter.cc +++ b/runtime/compiler_filter.cc @@ -33,6 +33,7 @@ bool CompilerFilter::IsBytecodeCompilationEnabled(Filter filter) { case CompilerFilter::kTime: case CompilerFilter::kSpeedProfile: case CompilerFilter::kSpeed: + case CompilerFilter::kLayoutProfile: case CompilerFilter::kEverythingProfile: case CompilerFilter::kEverything: return true; } @@ -52,6 +53,7 @@ bool CompilerFilter::IsJniCompilationEnabled(Filter filter) { case CompilerFilter::kTime: case CompilerFilter::kSpeedProfile: case CompilerFilter::kSpeed: + case CompilerFilter::kLayoutProfile: case CompilerFilter::kEverythingProfile: case CompilerFilter::kEverything: return true; } @@ -71,6 +73,7 @@ bool CompilerFilter::IsVerificationEnabled(Filter filter) { case CompilerFilter::kTime: case CompilerFilter::kSpeedProfile: case CompilerFilter::kSpeed: + case CompilerFilter::kLayoutProfile: case CompilerFilter::kEverythingProfile: case CompilerFilter::kEverything: return true; } @@ -97,6 +100,7 @@ bool CompilerFilter::DependsOnProfile(Filter filter) { case CompilerFilter::kVerifyProfile: case CompilerFilter::kSpaceProfile: case CompilerFilter::kSpeedProfile: + case CompilerFilter::kLayoutProfile: case CompilerFilter::kEverythingProfile: return true; } UNREACHABLE(); @@ -121,6 +125,7 @@ CompilerFilter::Filter CompilerFilter::GetNonProfileDependentFilterFrom(Filter f return CompilerFilter::kSpace; case CompilerFilter::kSpeedProfile: + case CompilerFilter::kLayoutProfile: return CompilerFilter::kSpeed; case CompilerFilter::kEverythingProfile: @@ -146,6 +151,7 @@ std::string CompilerFilter::NameOfFilter(Filter filter) { case CompilerFilter::kTime: return "time"; case CompilerFilter::kSpeedProfile: return "speed-profile"; case CompilerFilter::kSpeed: return "speed"; + case CompilerFilter::kLayoutProfile: return "layout-profile"; case CompilerFilter::kEverythingProfile: return "everything-profile"; case CompilerFilter::kEverything: return "everything"; } @@ -173,6 +179,8 @@ bool CompilerFilter::ParseCompilerFilter(const char* option, Filter* filter) { *filter = kSpeed; } else if (strcmp(option, "speed-profile") == 0) { *filter = kSpeedProfile; + } else if (strcmp(option, "layout-profile") == 0) { + *filter = kLayoutProfile; } else if (strcmp(option, "everything") == 0) { *filter = kEverything; } else if (strcmp(option, "everything-profile") == 0) { diff --git a/runtime/compiler_filter.h b/runtime/compiler_filter.h index 37631cc6d..781d43aa6 100644 --- a/runtime/compiler_filter.h +++ b/runtime/compiler_filter.h @@ -39,6 +39,7 @@ class CompilerFilter FINAL { kSpace, // Maximize space savings. kBalanced, // Good performance return on compilation investment. kSpeedProfile, // Maximize runtime performance based on profile. + kLayoutProfile, // Temporary filter for dexlayout. Will be merged with kSpeedProfile. kSpeed, // Maximize runtime performance. kEverythingProfile, // Compile everything capable of being compiled based on profile. kEverything, // Compile everything capable of being compiled. diff --git a/runtime/dex_file_verifier.cc b/runtime/dex_file_verifier.cc index 68e9f7392..061ea1a21 100644 --- a/runtime/dex_file_verifier.cc +++ b/runtime/dex_file_verifier.cc @@ -579,7 +579,7 @@ bool DexFileVerifier::CheckClassDataItemMethod(uint32_t idx, (reinterpret_cast(begin_ + header_->method_ids_off_) + idx)-> class_idx_; if (class_type_index != my_class_index) { - ErrorStringPrintf("Method's class index unexpected, %" PRIu16 "vs %" PRIu16, + ErrorStringPrintf("Method's class index unexpected, %" PRIu16 " vs %" PRIu16, my_class_index.index_, class_type_index.index_); return false; -- 2.11.0