From a6e81ed4c185b7362cd5199ebe5507d00883a9b0 Mon Sep 17 00:00:00 2001 From: Mathieu Chartier Date: Thu, 25 Feb 2016 13:52:10 -0800 Subject: [PATCH] Add lz4hc image compression format Smaller than lz4 and decompresses at the same speed. Compression is a bit slower. Example saves on old FB APK: Uncompressed: 44748800 bytes LZ4: 12443648 bytes LZ4HC: 11055104 bytes Generating the image slows down by ~1s per 20MB of image due to slower compression. Decompression is about the same speed but there should be a slight speedup since less data needs to be read from flash. Added test. Bug: 22858531 Change-Id: Ib2704305b9bec5b0ba3b1e871f59f4eedff330b7 --- compiler/image_test.cc | 5 +++++ compiler/image_writer.cc | 23 ++++++++++++++++++++--- dex2oat/dex2oat.cc | 4 +++- runtime/gc/space/image_space.cc | 10 +++++++++- runtime/image.h | 1 + runtime/oat_file_manager.cc | 19 +++++++++++-------- 6 files changed, 49 insertions(+), 13 deletions(-) diff --git a/compiler/image_test.cc b/compiler/image_test.cc index 992af2954..5763cec43 100644 --- a/compiler/image_test.cc +++ b/compiler/image_test.cc @@ -289,6 +289,11 @@ TEST_F(ImageTest, WriteReadLZ4) { TestWriteRead(ImageHeader::kStorageModeLZ4); } +TEST_F(ImageTest, WriteReadLZ4HC) { + TestWriteRead(ImageHeader::kStorageModeLZ4HC); +} + + TEST_F(ImageTest, ImageHeaderIsValid) { uint32_t image_begin = ART_BASE_ADDRESS; uint32_t image_size_ = 16 * KB; diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 5eff8f37e..871435b85 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -224,18 +225,28 @@ bool ImageWriter::Write(int image_fd, char* image_data = reinterpret_cast(image_info.image_->Begin()) + sizeof(ImageHeader); size_t data_size; const char* image_data_to_write; + const uint64_t compress_start_time = NanoTime(); CHECK_EQ(image_header->storage_mode_, image_storage_mode_); switch (image_storage_mode_) { case ImageHeader::kStorageModeLZ4: { - size_t compressed_max_size = LZ4_compressBound(image_data_size); + const size_t compressed_max_size = LZ4_compressBound(image_data_size); compressed_data.reset(new char[compressed_max_size]); data_size = LZ4_compress( reinterpret_cast(image_info.image_->Begin()) + sizeof(ImageHeader), &compressed_data[0], image_data_size); - image_data_to_write = &compressed_data[0]; - VLOG(compiler) << "Compressed from " << image_data_size << " to " << data_size; + + break; + } + case ImageHeader::kStorageModeLZ4HC: { + // Bound is same as non HC. + const size_t compressed_max_size = LZ4_compressBound(image_data_size); + compressed_data.reset(new char[compressed_max_size]); + data_size = LZ4_compressHC( + reinterpret_cast(image_info.image_->Begin()) + sizeof(ImageHeader), + &compressed_data[0], + image_data_size); break; } case ImageHeader::kStorageModeUncompressed: { @@ -249,6 +260,12 @@ bool ImageWriter::Write(int image_fd, } } + if (compressed_data != nullptr) { + image_data_to_write = &compressed_data[0]; + VLOG(compiler) << "Compressed from " << image_data_size << " to " << data_size << " in " + << PrettyDuration(NanoTime() - compress_start_time); + } + // Write header first, as uncompressed. image_header->data_size_ = data_size; if (!image_file->WriteFully(image_info.image_->Begin(), sizeof(ImageHeader))) { diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc index 53331284c..dfcb4bcae 100644 --- a/dex2oat/dex2oat.cc +++ b/dex2oat/dex2oat.cc @@ -217,7 +217,7 @@ NO_RETURN static void Usage(const char* fmt, ...) { UsageError(" --image=: specifies an output image filename."); UsageError(" Example: --image=/system/framework/boot.art"); UsageError(""); - UsageError(" --image-format=(uncompressed|lz4):"); + UsageError(" --image-format=(uncompressed|lz4|lz4hc):"); UsageError(" Which format to store the image."); UsageError(" Example: --image-format=lz4"); UsageError(" Default: uncompressed"); @@ -681,6 +681,8 @@ class Dex2Oat FINAL { const StringPiece format_str = option.substr(substr.length()); if (format_str == "lz4") { image_storage_mode_ = ImageHeader::kStorageModeLZ4; + } else if (format_str == "lz4hc") { + image_storage_mode_ = ImageHeader::kStorageModeLZ4HC; } else if (format_str == "uncompressed") { image_storage_mode_ = ImageHeader::kStorageModeUncompressed; } else { diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index bc21b334d..4ef36a449 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -1252,7 +1252,8 @@ ImageSpace* ImageSpace::Init(const char* image_filename, // Only care about the error message for the last address in addresses. We want to avoid the // overhead of printing the process maps if we can relocate. std::string* out_error_msg = (address == addresses.back()) ? &temp_error_msg : nullptr; - if (image_header->GetStorageMode() == ImageHeader::kStorageModeUncompressed) { + const ImageHeader::StorageMode storage_mode = image_header->GetStorageMode(); + if (storage_mode == ImageHeader::kStorageModeUncompressed) { map.reset(MemMap::MapFileAtAddress(address, image_header->GetImageSize(), PROT_READ | PROT_WRITE, @@ -1264,6 +1265,12 @@ ImageSpace* ImageSpace::Init(const char* image_filename, image_filename, /*out*/out_error_msg)); } else { + if (storage_mode != ImageHeader::kStorageModeLZ4 && + storage_mode != ImageHeader::kStorageModeLZ4HC) { + *error_msg = StringPrintf("Invalid storage mode in image header %d", + static_cast(storage_mode)); + return nullptr; + } // Reserve output and decompress into it. map.reset(MemMap::MapAnonymous(image_location, address, @@ -1289,6 +1296,7 @@ ImageSpace* ImageSpace::Init(const char* image_filename, } memcpy(map->Begin(), image_header, sizeof(ImageHeader)); const uint64_t start = NanoTime(); + // LZ4HC and LZ4 have same internal format, both use LZ4_decompress. TimingLogger::ScopedTiming timing2("LZ4 decompress image", &logger); const size_t decompressed_size = LZ4_decompress_safe( reinterpret_cast(temp_map->Begin()) + sizeof(ImageHeader), diff --git a/runtime/image.h b/runtime/image.h index 146ee00c8..8e5dbad57 100644 --- a/runtime/image.h +++ b/runtime/image.h @@ -81,6 +81,7 @@ class PACKED(4) ImageHeader { enum StorageMode : uint32_t { kStorageModeUncompressed, kStorageModeLZ4, + kStorageModeLZ4HC, kStorageModeCount, // Number of elements in enum. }; static constexpr StorageMode kDefaultStorageMode = kStorageModeUncompressed; diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc index 18cf81aa7..ea26d5876 100644 --- a/runtime/oat_file_manager.cc +++ b/runtime/oat_file_manager.cc @@ -16,6 +16,8 @@ #include "oat_file_manager.h" +#define ATRACE_TAG ATRACE_TAG_DALVIK +#include #include #include #include @@ -386,13 +388,15 @@ std::vector> OatFileManager::OpenDexFilesFromOat( ScopedSuspendAll ssa("Add image space"); runtime->GetHeap()->AddSpace(image_space.get()); } - added_image_space = true; - if (runtime->GetClassLinker()->AddImageSpace(image_space.get(), - h_loader, - dex_elements, - dex_location, - /*out*/&dex_files, - /*out*/&temp_error_msg)) { + ATRACE_BEGIN(StringPrintf("Adding image space for location %s", dex_location).c_str()); + added_image_space = runtime->GetClassLinker()->AddImageSpace(image_space.get(), + h_loader, + dex_elements, + dex_location, + /*out*/&dex_files, + /*out*/&temp_error_msg); + ATRACE_END(); + if (added_image_space) { // Successfully added image space to heap, release the map so that it does not get // freed. image_space.release(); @@ -407,7 +411,6 @@ std::vector> OatFileManager::OpenDexFilesFromOat( ScopedSuspendAll ssa("Remove image space"); runtime->GetHeap()->RemoveSpace(image_space.get()); } - added_image_space = false; // Non-fatal, don't update error_msg. } } -- 2.11.0