Add lz4hc image compression format

Smaller than lz4 and decompresses at the same speed. Compression is
a bit slower.

Example saves on old FB APK:
Uncompressed: 44748800 bytes
LZ4: 12443648 bytes
LZ4HC: 11055104 bytes

Generating the image slows down by ~1s per 20MB of image due to
slower compression. Decompression is about the same speed but there
should be a slight speedup since less data needs to be read from
flash.

Added test.

Bug: 22858531

Change-Id: Ib2704305b9bec5b0ba3b1e871f59f4eedff330b7

diff --git a/compiler/image_test.cc b/compiler/image_test.cc
index 992af29..5763cec 100644 (file)
--- a/compiler/image_test.cc
+++ b/compiler/image_test.cc
@@ -289,6 +289,11 @@ TEST_F(ImageTest, WriteReadLZ4) {
   TestWriteRead(ImageHeader::kStorageModeLZ4);
 }
 
+TEST_F(ImageTest, WriteReadLZ4HC) {
+  TestWriteRead(ImageHeader::kStorageModeLZ4HC);
+}
+
+
 TEST_F(ImageTest, ImageHeaderIsValid) {
     uint32_t image_begin = ART_BASE_ADDRESS;
     uint32_t image_size_ = 16 * KB;

diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc
index 5eff8f3..871435b 100644 (file)
--- a/compiler/image_writer.cc
+++ b/compiler/image_writer.cc
@@ -18,6 +18,7 @@
 
 #include <sys/stat.h>
 #include <lz4.h>
+#include <lz4hc.h>
 
 #include <memory>
 #include <numeric>
@@ -224,18 +225,28 @@ bool ImageWriter::Write(int image_fd,
     char* image_data = reinterpret_cast<char*>(image_info.image_->Begin()) + sizeof(ImageHeader);
     size_t data_size;
     const char* image_data_to_write;
+    const uint64_t compress_start_time = NanoTime();
 
     CHECK_EQ(image_header->storage_mode_, image_storage_mode_);
     switch (image_storage_mode_) {
       case ImageHeader::kStorageModeLZ4: {
-        size_t compressed_max_size = LZ4_compressBound(image_data_size);
+        const size_t compressed_max_size = LZ4_compressBound(image_data_size);
         compressed_data.reset(new char[compressed_max_size]);
         data_size = LZ4_compress(
             reinterpret_cast<char*>(image_info.image_->Begin()) + sizeof(ImageHeader),
             &compressed_data[0],
             image_data_size);
-        image_data_to_write = &compressed_data[0];
-        VLOG(compiler) << "Compressed from " << image_data_size << " to " << data_size;
+
+        break;
+      }
+      case ImageHeader::kStorageModeLZ4HC: {
+        // Bound is same as non HC.
+        const size_t compressed_max_size = LZ4_compressBound(image_data_size);
+        compressed_data.reset(new char[compressed_max_size]);
+        data_size = LZ4_compressHC(
+            reinterpret_cast<char*>(image_info.image_->Begin()) + sizeof(ImageHeader),
+            &compressed_data[0],
+            image_data_size);
         break;
       }
       case ImageHeader::kStorageModeUncompressed: {
@@ -249,6 +260,12 @@ bool ImageWriter::Write(int image_fd,
       }
     }
 
+    if (compressed_data != nullptr) {
+      image_data_to_write = &compressed_data[0];
+      VLOG(compiler) << "Compressed from " << image_data_size << " to " << data_size << " in "
+                     << PrettyDuration(NanoTime() - compress_start_time);
+    }
+
     // Write header first, as uncompressed.
     image_header->data_size_ = data_size;
     if (!image_file->WriteFully(image_info.image_->Begin(), sizeof(ImageHeader))) {

diff --git a/dex2oat/dex2oat.cc b/dex2oat/dex2oat.cc
index 5333128..dfcb4bc 100644 (file)
--- a/dex2oat/dex2oat.cc
+++ b/dex2oat/dex2oat.cc
@@ -217,7 +217,7 @@ NO_RETURN static void Usage(const char* fmt, ...) {
   UsageError("  --image=<file.art>: specifies an output image filename.");
   UsageError("      Example: --image=/system/framework/boot.art");
   UsageError("");
-  UsageError("  --image-format=(uncompressed|lz4):");
+  UsageError("  --image-format=(uncompressed|lz4|lz4hc):");
   UsageError("      Which format to store the image.");
   UsageError("      Example: --image-format=lz4");
   UsageError("      Default: uncompressed");
@@ -681,6 +681,8 @@ class Dex2Oat FINAL {
     const StringPiece format_str = option.substr(substr.length());
     if (format_str == "lz4") {
       image_storage_mode_ = ImageHeader::kStorageModeLZ4;
+    } else if (format_str == "lz4hc") {
+      image_storage_mode_ = ImageHeader::kStorageModeLZ4HC;
     } else if (format_str == "uncompressed") {
       image_storage_mode_ = ImageHeader::kStorageModeUncompressed;
     } else {

diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc
index bc21b33..4ef36a4 100644 (file)
--- a/runtime/gc/space/image_space.cc
+++ b/runtime/gc/space/image_space.cc
@@ -1252,7 +1252,8 @@ ImageSpace* ImageSpace::Init(const char* image_filename,
     // Only care about the error message for the last address in addresses. We want to avoid the
     // overhead of printing the process maps if we can relocate.
     std::string* out_error_msg = (address == addresses.back()) ? &temp_error_msg : nullptr;
-    if (image_header->GetStorageMode() == ImageHeader::kStorageModeUncompressed) {
+    const ImageHeader::StorageMode storage_mode = image_header->GetStorageMode();
+    if (storage_mode == ImageHeader::kStorageModeUncompressed) {
       map.reset(MemMap::MapFileAtAddress(address,
                                          image_header->GetImageSize(),
                                          PROT_READ | PROT_WRITE,
@@ -1264,6 +1265,12 @@ ImageSpace* ImageSpace::Init(const char* image_filename,
                                          image_filename,
                                          /*out*/out_error_msg));
     } else {
+      if (storage_mode != ImageHeader::kStorageModeLZ4 &&
+          storage_mode != ImageHeader::kStorageModeLZ4HC) {
+        *error_msg = StringPrintf("Invalid storage mode in image header %d",
+                                  static_cast<int>(storage_mode));
+        return nullptr;
+      }
       // Reserve output and decompress into it.
       map.reset(MemMap::MapAnonymous(image_location,
                                      address,
@@ -1289,6 +1296,7 @@ ImageSpace* ImageSpace::Init(const char* image_filename,
         }
         memcpy(map->Begin(), image_header, sizeof(ImageHeader));
         const uint64_t start = NanoTime();
+        // LZ4HC and LZ4 have same internal format, both use LZ4_decompress.
         TimingLogger::ScopedTiming timing2("LZ4 decompress image", &logger);
         const size_t decompressed_size = LZ4_decompress_safe(
             reinterpret_cast<char*>(temp_map->Begin()) + sizeof(ImageHeader),

diff --git a/runtime/image.h b/runtime/image.h
index 146ee00..8e5dbad 100644 (file)
--- a/runtime/image.h
+++ b/runtime/image.h
@@ -81,6 +81,7 @@ class PACKED(4) ImageHeader {
   enum StorageMode : uint32_t {
     kStorageModeUncompressed,
     kStorageModeLZ4,
+    kStorageModeLZ4HC,
     kStorageModeCount,  // Number of elements in enum.
   };
   static constexpr StorageMode kDefaultStorageMode = kStorageModeUncompressed;

diff --git a/runtime/oat_file_manager.cc b/runtime/oat_file_manager.cc
index 18cf81a..ea26d58 100644 (file)
--- a/runtime/oat_file_manager.cc
+++ b/runtime/oat_file_manager.cc
@@ -16,6 +16,8 @@
 
 #include "oat_file_manager.h"
 
+#define ATRACE_TAG ATRACE_TAG_DALVIK
+#include <cutils/trace.h>
 #include <memory>
 #include <queue>
 #include <vector>
@@ -386,13 +388,15 @@ std::vector<std::unique_ptr<const DexFile>> OatFileManager::OpenDexFilesFromOat(
             ScopedSuspendAll ssa("Add image space");
             runtime->GetHeap()->AddSpace(image_space.get());
           }
-          added_image_space = true;
-          if (runtime->GetClassLinker()->AddImageSpace(image_space.get(),
-                                                       h_loader,
-                                                       dex_elements,
-                                                       dex_location,
-                                                       /*out*/&dex_files,
-                                                       /*out*/&temp_error_msg)) {
+          ATRACE_BEGIN(StringPrintf("Adding image space for location %s", dex_location).c_str());
+          added_image_space = runtime->GetClassLinker()->AddImageSpace(image_space.get(),
+                                                                       h_loader,
+                                                                       dex_elements,
+                                                                       dex_location,
+                                                                       /*out*/&dex_files,
+                                                                       /*out*/&temp_error_msg);
+          ATRACE_END();
+          if (added_image_space) {
             // Successfully added image space to heap, release the map so that it does not get
             // freed.
             image_space.release();
@@ -407,7 +411,6 @@ std::vector<std::unique_ptr<const DexFile>> OatFileManager::OpenDexFilesFromOat(
               ScopedSuspendAll ssa("Remove image space");
               runtime->GetHeap()->RemoveSpace(image_space.get());
             }
-            added_image_space = false;
             // Non-fatal, don't update error_msg.
           }
         }