From 5fad2039bbd4e55d671106c8b39a6e451b85ac23 Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Mon, 24 Feb 2014 08:13:45 -0800 Subject: [PATCH] handle don't care regions in the system image The system partitions has regions that we shouldn't write and can't depend on the contents of. Adds a new script to generate a map of these regions (using the sparse image as input), and include the map in the package zip so it can be used when writing or patching the system partition. Also fixes a bug where the wrong SELinux file contexts are used when generating incrementals. Change-Id: Iaca5b967a3b7d1df843c7c21becc19b3f1633dad --- core/Makefile | 3 +- tools/releasetools/build_image.py | 9 ++ tools/releasetools/common.py | 2 +- tools/releasetools/edify_generator.py | 29 ++++-- tools/releasetools/img_from_target_files.py | 23 ++++- tools/releasetools/ota_from_target_files | 33 +++++-- tools/releasetools/simg_map.py | 148 ++++++++++++++++++++++++++++ 7 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 tools/releasetools/simg_map.py diff --git a/core/Makefile b/core/Makefile index e7eafe8bb..c0b2902e5 100644 --- a/core/Makefile +++ b/core/Makefile @@ -1135,7 +1135,8 @@ DISTTOOLS := $(HOST_OUT_EXECUTABLES)/minigzip \ $(HOST_OUT_EXECUTABLES)/mkuserimg.sh \ $(HOST_OUT_EXECUTABLES)/make_ext4fs \ $(HOST_OUT_EXECUTABLES)/simg2img \ - $(HOST_OUT_EXECUTABLES)/e2fsck + $(HOST_OUT_EXECUTABLES)/e2fsck \ + $(HOST_OUT_EXECUTABLES)/xdelta3 OTATOOLS := $(DISTTOOLS) \ $(HOST_OUT_EXECUTABLES)/aapt diff --git a/tools/releasetools/build_image.py b/tools/releasetools/build_image.py index ffda6cf16..927d89af0 100755 --- a/tools/releasetools/build_image.py +++ b/tools/releasetools/build_image.py @@ -27,6 +27,8 @@ import sys import commands import shutil +import simg_map + def RunCommand(cmd): """ Echo and run the given command @@ -146,6 +148,13 @@ def UnsparseImage(sparse_image_path, replace=True): return False, None return True, unsparse_image_path +def MappedUnsparseImage(sparse_image_path, unsparse_image_path, + map_path, mapped_unsparse_image_path): + if simg_map.ComputeMap(sparse_image_path, unsparse_image_path, + map_path, mapped_unsparse_image_path): + return False + return True + def MakeVerityEnabledImage(out_file, prop_dict): """Creates an image that is verifiable using dm-verity. diff --git a/tools/releasetools/common.py b/tools/releasetools/common.py index b7e18fa17..ca73ee437 100644 --- a/tools/releasetools/common.py +++ b/tools/releasetools/common.py @@ -1015,7 +1015,7 @@ def MakeSystemPatch(source_file, target_file): with open(output_file.name + ".xz") as patch_file: patch_data = patch_file.read() os.unlink(patch_file.name) - return File("system.img.p", patch_data) + return File("system.muimg.p", patch_data) def MakeRecoveryPatch(input_dir, output_sink, recovery_img, boot_img, info_dict=None): diff --git a/tools/releasetools/edify_generator.py b/tools/releasetools/edify_generator.py index 797806221..af545dba8 100644 --- a/tools/releasetools/edify_generator.py +++ b/tools/releasetools/edify_generator.py @@ -190,6 +190,16 @@ class EdifyGenerator(object): (p.fs_type, common.PARTITION_TYPES[p.fs_type], p.device, p.length, p.mount_point)) + def WipeBlockDevice(self, partition): + if partition != "/system": + raise ValueError(("WipeBlockDevice currently only works " + "on /system, not %s\n") % (partition,)) + fstab = self.info.get("fstab", None) + size = self.info.get("system_size", None) + device = fstab[partition].device + + self.script.append('wipe_block_device("%s", %s);' % (device, size)) + def DeleteFiles(self, file_list): """Delete all files in file_list.""" if not file_list: return @@ -224,7 +234,7 @@ class EdifyGenerator(object): cmd = "".join(cmd) self.script.append(self._WordWrap(cmd)) - def WriteRawImage(self, mount_point, fn): + def WriteRawImage(self, mount_point, fn, mapfn=None): """Write the given package file into the partition for the given mount point.""" @@ -238,8 +248,13 @@ class EdifyGenerator(object): 'write_raw_image(package_extract_file("%(fn)s"), "%(device)s");' % args) elif partition_type == "EMMC": - self.script.append( - 'package_extract_file("%(fn)s", "%(device)s");' % args) + if mapfn: + args["map"] = mapfn + self.script.append( + 'package_extract_file("%(fn)s", "%(device)s", "%(map)s");' % args) + else: + self.script.append( + 'package_extract_file("%(fn)s", "%(device)s");' % args) else: raise ValueError("don't know how to write \"%s\" partitions" % (p.fs_type,)) @@ -309,7 +324,9 @@ class EdifyGenerator(object): common.ZipWriteStr(output_zip, "META-INF/com/google/android/update-binary", data, perms=0755) - def Syspatch(self, filename, size, target_sha, source_sha, patchfile): + def Syspatch(self, filename, target_mapfile, target_sha, + source_mapfile, source_sha, patchfile): """Applies a compressed binary patch to a block device.""" - call = 'syspatch("%s", "%s", "%s", "%s", "%s");' - self.script.append(call % (filename, size, target_sha, source_sha, patchfile)) + call = 'syspatch("%s", "%s", "%s", "%s", "%s", "%s");' + self.script.append(call % (filename, target_mapfile, target_sha, + source_mapfile, source_sha, patchfile)) diff --git a/tools/releasetools/img_from_target_files.py b/tools/releasetools/img_from_target_files.py index bd11a45a1..596a47e37 100755 --- a/tools/releasetools/img_from_target_files.py +++ b/tools/releasetools/img_from_target_files.py @@ -60,7 +60,7 @@ def AddSystem(output_zip, sparse=True): common.ZipWriteStr(output_zip, "system.img", data) -def BuildSystem(input_dir, info_dict, sparse=True): +def BuildSystem(input_dir, info_dict, sparse=True, map_file=None): print "creating system.img..." img = tempfile.NamedTemporaryFile() @@ -87,6 +87,8 @@ def BuildSystem(input_dir, info_dict, sparse=True): image_props, img.name) assert succ, "build system.img image failed" + mapdata = None + if sparse: img.seek(os.SEEK_SET, 0) data = img.read() @@ -95,13 +97,30 @@ def BuildSystem(input_dir, info_dict, sparse=True): success, name = build_image.UnsparseImage(img.name, replace=False) if not success: assert False, "unsparsing system.img failed" + + if map_file: + mmap = tempfile.NamedTemporaryFile() + mimg = tempfile.NamedTemporaryFile(delete=False) + success = build_image.MappedUnsparseImage( + img.name, name, mmap.name, mimg.name) + if not success: + assert False, "creating sparse map failed" + os.unlink(name) + name = mimg.name + + with open(mmap.name) as f: + mapdata = f.read() + try: with open(name) as f: data = f.read() finally: os.unlink(name) - return data + if mapdata is None: + return data + else: + return mapdata, data def AddVendor(output_zip): diff --git a/tools/releasetools/ota_from_target_files b/tools/releasetools/ota_from_target_files index a7936b13a..59dd06e7f 100755 --- a/tools/releasetools/ota_from_target_files +++ b/tools/releasetools/ota_from_target_files @@ -461,8 +461,14 @@ else if get_stage("%(bcb_dev)s", "stage") == "3/3" then script.ShowProgress(system_progress, 30) if block_based: - img_from_target_files.AddSystem(output_zip, sparse=False) - script.WriteRawImage("/system", "system.img") + mapdata, data = img_from_target_files.BuildSystem( + OPTIONS.input_tmp, OPTIONS.info_dict, + sparse=False, map_file=True) + + common.ZipWriteStr(output_zip, "system.map", mapdata) + common.ZipWriteStr(output_zip, "system.muimg", data) + script.WipeBlockDevice("/system") + script.WriteRawImage("/system", "system.muimg", mapfn="system.map") else: script.FormatPartition("/system") script.Mount("/system") @@ -608,8 +614,10 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): with tempfile.NamedTemporaryFile() as tgt_file: print "building source system image..." src_file = tempfile.NamedTemporaryFile() - src_data = img_from_target_files.BuildSystem( - OPTIONS.source_tmp, OPTIONS.source_info_dict, sparse=False) + src_mapdata, src_data = img_from_target_files.BuildSystem( + OPTIONS.source_tmp, OPTIONS.source_info_dict, + sparse=False, map_file=True) + src_sys_sha1 = sha1(src_data).hexdigest() print "source system sha1:", src_sys_sha1 src_file.write(src_data) @@ -617,8 +625,9 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): print "building target system image..." tgt_file = tempfile.NamedTemporaryFile() - tgt_data = img_from_target_files.BuildSystem( - OPTIONS.target_tmp, OPTIONS.target_info_dict, sparse=False) + tgt_mapdata, tgt_data = img_from_target_files.BuildSystem( + OPTIONS.target_tmp, OPTIONS.target_info_dict, + sparse=False, map_file=True) tgt_sys_sha1 = sha1(tgt_data).hexdigest() print "target system sha1:", tgt_sys_sha1 tgt_sys_len = len(tgt_data) @@ -628,6 +637,10 @@ def WriteBlockIncrementalOTAPackage(target_zip, source_zip, output_zip): system_type, system_device = common.GetTypeAndDevice("/system", OPTIONS.info_dict) system_patch = common.MakeSystemPatch(src_file, tgt_file) system_patch.AddToZip(output_zip, compression=zipfile.ZIP_STORED) + src_mapfilename = system_patch.name + ".src.map" + common.ZipWriteStr(output_zip, src_mapfilename, src_mapdata) + tgt_mapfilename = system_patch.name + ".tgt.map" + common.ZipWriteStr(output_zip, tgt_mapfilename, tgt_mapdata) AppendAssertions(script, OPTIONS.target_info_dict) device_specific.IncrementalOTA_Assertions() @@ -713,9 +726,8 @@ else script.Print("Patching system image...") script.Syspatch(system_device, - OPTIONS.info_dict["system_size"], - tgt_sys_sha1, - src_sys_sha1, + tgt_mapfilename, tgt_sys_sha1, + src_mapfilename, src_sys_sha1, system_patch.name) if OPTIONS.two_step: @@ -1249,6 +1261,9 @@ def main(argv): OPTIONS.source_tmp, source_zip = common.UnzipTemp(OPTIONS.incremental_source) OPTIONS.target_info_dict = OPTIONS.info_dict OPTIONS.source_info_dict = common.LoadInfoDict(source_zip) + if "selinux_fc" in OPTIONS.source_info_dict: + OPTIONS.source_info_dict["selinux_fc"] = os.path.join(OPTIONS.source_tmp, "BOOT", "RAMDISK", + "file_contexts") if OPTIONS.package_key is None: OPTIONS.package_key = OPTIONS.source_info_dict.get( "default_system_dev_certificate", diff --git a/tools/releasetools/simg_map.py b/tools/releasetools/simg_map.py new file mode 100644 index 000000000..22dc8635d --- /dev/null +++ b/tools/releasetools/simg_map.py @@ -0,0 +1,148 @@ +#! /usr/bin/env python + +# Copyright (C) 2012 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import getopt, posixpath, signal, struct, sys + +def main(): + if len(sys.argv) == 4: + print("No sparse_image_file specified") + usage(me) + + sparse_fn = sys.argv[1] + unsparse_fn = sys.argv[2] + map_file = sys.argv[3] + mapped_unsparse_fn = sys.argv[4] + + return ComputeMap(sparse_fn, unsparse_fn, map_file, mapped_unsparse_fn) + + +def ComputeMap(sparse_fn, unsparse_fn, map_file, mapped_unsparse_fn): + care_map = [] + + with open(sparse_fn, "rb") as FH: + header_bin = FH.read(28) + header = struct.unpack("