Use soong_zip and zip2zip to make dist faster

soong_zip parallelizes the compression when creating a zip file, which
makes these packaging steps far shorter.

zip2zip skips the decompression and recompression during the -img- file
creation.

For an aosp_arm64-eng aosp/master build:

target_files.zip: 92s to 60s
symbols.zip: 147s to 7s
img.zip: 64s to 0.5s

There's still room to parallelize the image compression during
target_files.zip (add_img_to_target_files step takes most of the time)

Change-Id: I7b6a91e4a7dbeda2e49ca936b10181cff2f973d7

diff --git a/core/Makefile b/core/Makefile
index e763c73..8885172 100644 (file)
--- a/core/Makefile
+++ b/core/Makefile
@@ -1733,11 +1733,12 @@ $(BUILT_TARGET_FILES_PACKAGE): \
                $(INSTALLED_ANDROID_INFO_TXT_TARGET) \
                $(SELINUX_FC) \
                $(APKCERTS_FILE) \
+               $(SOONG_ZIP) \
                $(HOST_OUT_EXECUTABLES)/fs_config \
                build/tools/releasetools/add_img_to_target_files \
                | $(ACP)
        @echo "Package target files: $@"
-       $(hide) rm -rf $@ $(zip_root)
+       $(hide) rm -rf $@ $@.list $(zip_root)
        $(hide) mkdir -p $(dir $@) $(zip_root)
 ifneq (,$(INSTALLED_RECOVERYIMAGE_TARGET)$(filter true,$(BOARD_USES_RECOVERY_AS_BOOT)))
        @# Components of the recovery image
@@ -1929,9 +1930,9 @@ ifdef BOARD_PREBUILT_VENDORIMAGE
 endif
        @# Zip everything up, preserving symlinks and placing META/ files first to
        @# help early validation of the .zip file while uploading it.
-       $(hide) (cd $(zip_root) && \
-               zip -qryX ../$(notdir $@) ./META && \
-               zip -qryXu ../$(notdir $@) .)
+       $(hide) find $(zip_root)/META | sort >$@.list
+       $(hide) find $(zip_root) | grep -v "^$(zip_root)/META/" | sort >>$@.list
+       $(hide) $(SOONG_ZIP) -d -o $@ -C $(zip_root) -l $@.list
        @# Run fs_config on all the system, vendor, boot ramdisk,
        @# and recovery ramdisk files in the zip, and save the output
        $(hide) zipinfo -1 $@ | awk 'BEGIN { FS="SYSTEM/" } /^SYSTEM\// {print "system/" $$2}' | $(HOST_OUT_EXECUTABLES)/fs_config -C -D $(TARGET_OUT) -S $(SELINUX_FC) > $(zip_root)/META/filesystem_config.txt
@@ -1995,13 +1996,10 @@ name := $(name)-img-$(FILE_NAME_TAG)
 
 INTERNAL_UPDATE_PACKAGE_TARGET := $(PRODUCT_OUT)/$(name).zip
 
-$(INTERNAL_UPDATE_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) \
-               build/tools/releasetools/img_from_target_files
+$(INTERNAL_UPDATE_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) $(ZIP2ZIP)
        @echo "Package: $@"
-       $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
-          ./build/tools/releasetools/img_from_target_files -v \
-          -p $(HOST_OUT) \
-          $(BUILT_TARGET_FILES_PACKAGE) $@
+       $(hide) $(ZIP2ZIP) -i $(BUILT_TARGET_FILES_PACKAGE) -o $@ \
+          OTA/android-info.txt:android-info.txt "IMAGES/*.img:."
 
 .PHONY: updatepackage
 updatepackage: $(INTERNAL_UPDATE_PACKAGE_TARGET)
@@ -2025,11 +2023,13 @@ $(SYMBOLS_ZIP): $(INSTALLED_SYSTEMIMAGE) \
                $(INSTALLED_VENDORIMAGE_TARGET) \
                $(updater_dep)
 endif
-$(SYMBOLS_ZIP):
+$(SYMBOLS_ZIP): PRIVATE_LIST_FILE := $(call intermediates-dir-for,PACKAGING,symbols)/filelist
+$(SYMBOLS_ZIP): $(SOONG_ZIP)
        @echo "Package symbols: $@"
-       $(hide) rm -rf $@
-       $(hide) mkdir -p $(dir $@) $(TARGET_OUT_UNSTRIPPED)
-       $(hide) zip -qrX $@ $(TARGET_OUT_UNSTRIPPED)
+       $(hide) rm -rf $@ $(PRIVATE_LIST_FILE)
+       $(hide) mkdir -p $(dir $@) $(TARGET_OUT_UNSTRIPPED) $(dir $(PRIVATE_LIST_FILE))
+       $(hide) find $(TARGET_OUT_UNSTRIPPED) | sort >$(PRIVATE_LIST_FILE)
+       $(hide) $(SOONG_ZIP) -d -o $@ -C $(TARGET_OUT_UNSTRIPPED) -l $(PRIVATE_LIST_FILE)
 
 # -----------------------------------------------------------------
 # A zip of the Android Apps. Not keeping full path so that we don't

diff --git a/core/config.mk b/core/config.mk
index 4ad212f..4f4b742 100644 (file)
--- a/core/config.mk
+++ b/core/config.mk
@@ -472,6 +472,9 @@ BCC_COMPAT := $(HOST_OUT_EXECUTABLES)/bcc_compat
 DX := $(HOST_OUT_EXECUTABLES)/dx
 MAINDEXCLASSES := $(HOST_OUT_EXECUTABLES)/mainDexClasses
 
+SOONG_ZIP := $(SOONG_HOST_OUT_EXECUTABLES)/soong_zip
+ZIP2ZIP := $(SOONG_HOST_OUT_EXECUTABLES)/zip2zip
+
 # Always use prebuilts for ckati and makeparallel
 prebuilt_build_tools := prebuilts/build-tools
 prebuilt_build_tools_bin := $(prebuilt_build_tools)/$(HOST_PREBUILT_TAG)/bin