tools/releasetools/build_image.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2011 The Android Open Source Project
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """
  18 Build image output_image_file from input_directory and properties_file.
  19
  20 Usage:  build_image input_directory properties_file output_image_file
  21
  22 """
  23 import os
  24 import os.path
  25 import re
  26 import subprocess
  27 import sys
  28 import commands
  29 import common
  30 import shutil
  31 import tempfile
  32
  33 OPTIONS = common.OPTIONS
  34
  35 FIXED_SALT = "aee087a5be3b982978c923f566a94613496b417f2af592639bc80d141e34dfe7"
  36
  37 def RunCommand(cmd):
  38   """Echo and run the given command.
  39
  40   Args:
  41     cmd: the command represented as a list of strings.
  42   Returns:
  43     A tuple of the output and the exit code.
  44   """
  45   print "Running: ", " ".join(cmd)
  46   p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  47   output, _ = p.communicate()
  48   print "%s" % (output.rstrip(),)
  49   return (output, p.returncode)
  50
  51 def GetVerityTreeSize(partition_size):
  52   cmd = "build_verity_tree -s %d"
  53   cmd %= partition_size
  54   status, output = commands.getstatusoutput(cmd)
  55   if status:
  56     print output
  57     return False, 0
  58   return True, int(output)
  59
  60 def GetVerityMetadataSize(partition_size):
  61   cmd = "system/extras/verity/build_verity_metadata.py -s %d"
  62   cmd %= partition_size
  63
  64   status, output = commands.getstatusoutput(cmd)
  65   if status:
  66     print output
  67     return False, 0
  68   return True, int(output)
  69
  70 def AdjustPartitionSizeForVerity(partition_size):
  71   """Modifies the provided partition size to account for the verity metadata.
  72
  73   This information is used to size the created image appropriately.
  74   Args:
  75     partition_size: the size of the partition to be verified.
  76   Returns:
  77     The size of the partition adjusted for verity metadata.
  78   """
  79   success, verity_tree_size = GetVerityTreeSize(partition_size)
  80   if not success:
  81     return 0
  82   success, verity_metadata_size = GetVerityMetadataSize(partition_size)
  83   if not success:
  84     return 0
  85   return partition_size - verity_tree_size - verity_metadata_size
  86
  87 def BuildVerityTree(sparse_image_path, verity_image_path, prop_dict):
  88   cmd = "build_verity_tree -A %s %s %s" % (
  89       FIXED_SALT, sparse_image_path, verity_image_path)
  90   print cmd
  91   status, output = commands.getstatusoutput(cmd)
  92   if status:
  93     print "Could not build verity tree! Error: %s" % output
  94     return False
  95   root, salt = output.split()
  96   prop_dict["verity_root_hash"] = root
  97   prop_dict["verity_salt"] = salt
  98   return True
  99
 100 def BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt,
 101                         block_device, signer_path, key):
 102   cmd_template = (
 103       "system/extras/verity/build_verity_metadata.py %s %s %s %s %s %s %s")
 104   cmd = cmd_template % (image_size, verity_metadata_path, root_hash, salt,
 105                         block_device, signer_path, key)
 106   print cmd
 107   status, output = commands.getstatusoutput(cmd)
 108   if status:
 109     print "Could not build verity metadata! Error: %s" % output
 110     return False
 111   return True
 112
 113 def Append2Simg(sparse_image_path, unsparse_image_path, error_message):
 114   """Appends the unsparse image to the given sparse image.
 115
 116   Args:
 117     sparse_image_path: the path to the (sparse) image
 118     unsparse_image_path: the path to the (unsparse) image
 119   Returns:
 120     True on success, False on failure.
 121   """
 122   cmd = "append2simg %s %s"
 123   cmd %= (sparse_image_path, unsparse_image_path)
 124   print cmd
 125   status, output = commands.getstatusoutput(cmd)
 126   if status:
 127     print "%s: %s" % (error_message, output)
 128     return False
 129   return True
 130
 131 def BuildVerifiedImage(data_image_path, verity_image_path,
 132                        verity_metadata_path):
 133   if not Append2Simg(data_image_path, verity_metadata_path,
 134                      "Could not append verity metadata!"):
 135     return False
 136   if not Append2Simg(data_image_path, verity_image_path,
 137                      "Could not append verity tree!"):
 138     return False
 139   return True
 140
 141 def UnsparseImage(sparse_image_path, replace=True):
 142   img_dir = os.path.dirname(sparse_image_path)
 143   unsparse_image_path = "unsparse_" + os.path.basename(sparse_image_path)
 144   unsparse_image_path = os.path.join(img_dir, unsparse_image_path)
 145   if os.path.exists(unsparse_image_path):
 146     if replace:
 147       os.unlink(unsparse_image_path)
 148     else:
 149       return True, unsparse_image_path
 150   inflate_command = ["simg2img", sparse_image_path, unsparse_image_path]
 151   (_, exit_code) = RunCommand(inflate_command)
 152   if exit_code != 0:
 153     os.remove(unsparse_image_path)
 154     return False, None
 155   return True, unsparse_image_path
 156
 157 def MakeVerityEnabledImage(out_file, prop_dict):
 158   """Creates an image that is verifiable using dm-verity.
 159
 160   Args:
 161     out_file: the location to write the verifiable image at
 162     prop_dict: a dictionary of properties required for image creation and
 163                verification
 164   Returns:
 165     True on success, False otherwise.
 166   """
 167   # get properties
 168   image_size = prop_dict["partition_size"]
 169   block_dev = prop_dict["verity_block_device"]
 170   signer_key = prop_dict["verity_key"] + ".pk8"
 171   if OPTIONS.verity_signer_path is not None:
 172     signer_path = OPTIONS.verity_signer_path + ' '
 173     signer_path += ' '.join(OPTIONS.verity_signer_args)
 174   else:
 175     signer_path = prop_dict["verity_signer_cmd"]
 176
 177   # make a tempdir
 178   tempdir_name = tempfile.mkdtemp(suffix="_verity_images")
 179
 180   # get partial image paths
 181   verity_image_path = os.path.join(tempdir_name, "verity.img")
 182   verity_metadata_path = os.path.join(tempdir_name, "verity_metadata.img")
 183
 184   # build the verity tree and get the root hash and salt
 185   if not BuildVerityTree(out_file, verity_image_path, prop_dict):
 186     shutil.rmtree(tempdir_name, ignore_errors=True)
 187     return False
 188
 189   # build the metadata blocks
 190   root_hash = prop_dict["verity_root_hash"]
 191   salt = prop_dict["verity_salt"]
 192   if not BuildVerityMetadata(image_size, verity_metadata_path, root_hash, salt,
 193                              block_dev, signer_path, signer_key):
 194     shutil.rmtree(tempdir_name, ignore_errors=True)
 195     return False
 196
 197   # build the full verified image
 198   if not BuildVerifiedImage(out_file,
 199                             verity_image_path,
 200                             verity_metadata_path):
 201     shutil.rmtree(tempdir_name, ignore_errors=True)
 202     return False
 203
 204   shutil.rmtree(tempdir_name, ignore_errors=True)
 205   return True
 206
 207 def BuildImage(in_dir, prop_dict, out_file, target_out=None):
 208   """Build an image to out_file from in_dir with property prop_dict.
 209
 210   Args:
 211     in_dir: path of input directory.
 212     prop_dict: property dictionary.
 213     out_file: path of the output image file.
 214     target_out: path of the product out directory to read device specific FS config files.
 215
 216   Returns:
 217     True iff the image is built successfully.
 218   """
 219   # system_root_image=true: build a system.img that combines the contents of
 220   # /system and the ramdisk, and can be mounted at the root of the file system.
 221   origin_in = in_dir
 222   fs_config = prop_dict.get("fs_config")
 223   if (prop_dict.get("system_root_image") == "true"
 224       and prop_dict["mount_point"] == "system"):
 225     in_dir = tempfile.mkdtemp()
 226     # Change the mount point to "/"
 227     prop_dict["mount_point"] = "/"
 228     if fs_config:
 229       # We need to merge the fs_config files of system and ramdisk.
 230       fd, merged_fs_config = tempfile.mkstemp(prefix="root_fs_config",
 231                                               suffix=".txt")
 232       os.close(fd)
 233       with open(merged_fs_config, "w") as fw:
 234         if "ramdisk_fs_config" in prop_dict:
 235           with open(prop_dict["ramdisk_fs_config"]) as fr:
 236             fw.writelines(fr.readlines())
 237         with open(fs_config) as fr:
 238           fw.writelines(fr.readlines())
 239       fs_config = merged_fs_config
 240
 241   build_command = []
 242   fs_type = prop_dict.get("fs_type", "")
 243   run_fsck = False
 244
 245   fs_spans_partition = True
 246   if fs_type.startswith("squash"):
 247     fs_spans_partition = False
 248
 249   is_verity_partition = "verity_block_device" in prop_dict
 250   verity_supported = prop_dict.get("verity") == "true"
 251   # Adjust the partition size to make room for the hashes if this is to be
 252   # verified.
 253   if verity_supported and is_verity_partition and fs_spans_partition:
 254     partition_size = int(prop_dict.get("partition_size"))
 255
 256     adjusted_size = AdjustPartitionSizeForVerity(partition_size)
 257     if not adjusted_size:
 258       return False
 259     prop_dict["partition_size"] = str(adjusted_size)
 260     prop_dict["original_partition_size"] = str(partition_size)
 261
 262   if fs_type.startswith("ext"):
 263     build_command = ["mkuserimg.sh"]
 264     if "extfs_sparse_flag" in prop_dict:
 265       build_command.append(prop_dict["extfs_sparse_flag"])
 266       run_fsck = True
 267     build_command.extend([in_dir, out_file, fs_type,
 268                           prop_dict["mount_point"]])
 269     build_command.append(prop_dict["partition_size"])
 270     if "journal_size" in prop_dict:
 271       build_command.extend(["-j", prop_dict["journal_size"]])
 272     if "timestamp" in prop_dict:
 273       build_command.extend(["-T", str(prop_dict["timestamp"])])
 274     if fs_config:
 275       build_command.extend(["-C", fs_config])
 276     if target_out:
 277       build_command.extend(["-D", target_out])
 278     if "block_list" in prop_dict:
 279       build_command.extend(["-B", prop_dict["block_list"]])
 280     build_command.extend(["-L", prop_dict["mount_point"]])
 281     if "selinux_fc" in prop_dict:
 282       build_command.append(prop_dict["selinux_fc"])
 283   elif fs_type.startswith("squash"):
 284     build_command = ["mksquashfsimage.sh"]
 285     build_command.extend([in_dir, out_file])
 286     build_command.extend(["-s"])
 287     build_command.extend(["-m", prop_dict["mount_point"]])
 288     if target_out:
 289       build_command.extend(["-d", target_out])
 290     if "selinux_fc" in prop_dict:
 291       build_command.extend(["-c", prop_dict["selinux_fc"]])
 292     if "squashfs_compressor" in prop_dict:
 293       build_command.extend(["-z", prop_dict["squashfs_compressor"]])
 294     if "squashfs_compressor_opt" in prop_dict:
 295       build_command.extend(["-zo", prop_dict["squashfs_compressor_opt"]])
 296   elif fs_type.startswith("f2fs"):
 297     build_command = ["mkf2fsuserimg.sh"]
 298     build_command.extend([out_file, prop_dict["partition_size"]])
 299   else:
 300     build_command = ["mkyaffs2image", "-f"]
 301     if prop_dict.get("mkyaffs2_extra_flags", None):
 302       build_command.extend(prop_dict["mkyaffs2_extra_flags"].split())
 303     build_command.append(in_dir)
 304     build_command.append(out_file)
 305     if "selinux_fc" in prop_dict:
 306       build_command.append(prop_dict["selinux_fc"])
 307       build_command.append(prop_dict["mount_point"])
 308
 309   if in_dir != origin_in:
 310     # Construct a staging directory of the root file system.
 311     ramdisk_dir = prop_dict.get("ramdisk_dir")
 312     if ramdisk_dir:
 313       shutil.rmtree(in_dir)
 314       shutil.copytree(ramdisk_dir, in_dir, symlinks=True)
 315     staging_system = os.path.join(in_dir, "system")
 316     shutil.rmtree(staging_system, ignore_errors=True)
 317     shutil.copytree(origin_in, staging_system, symlinks=True)
 318
 319   reserved_blocks = prop_dict.get("has_ext4_reserved_blocks") == "true"
 320   ext4fs_output = None
 321
 322   try:
 323     if reserved_blocks and fs_type.startswith("ext4"):
 324       (ext4fs_output, exit_code) = RunCommand(build_command)
 325     else:
 326       (_, exit_code) = RunCommand(build_command)
 327   finally:
 328     if in_dir != origin_in:
 329       # Clean up temporary directories and files.
 330       shutil.rmtree(in_dir, ignore_errors=True)
 331       if fs_config:
 332         os.remove(fs_config)
 333   if exit_code != 0:
 334     return False
 335
 336   # Bug: 21522719, 22023465
 337   # There are some reserved blocks on ext4 FS (lesser of 4096 blocks and 2%).
 338   # We need to deduct those blocks from the available space, since they are
 339   # not writable even with root privilege. It only affects devices using
 340   # file-based OTA and a kernel version of 3.10 or greater (currently just
 341   # sprout).
 342   if reserved_blocks and fs_type.startswith("ext4"):
 343     assert ext4fs_output is not None
 344     ext4fs_stats = re.compile(
 345         r'Created filesystem with .* (?P<used_blocks>[0-9]+)/'
 346         r'(?P<total_blocks>[0-9]+) blocks')
 347     m = ext4fs_stats.match(ext4fs_output.strip().split('\n')[-1])
 348     used_blocks = int(m.groupdict().get('used_blocks'))
 349     total_blocks = int(m.groupdict().get('total_blocks'))
 350     reserved_blocks = min(4096, int(total_blocks * 0.02))
 351     adjusted_blocks = total_blocks - reserved_blocks
 352     if used_blocks > adjusted_blocks:
 353       mount_point = prop_dict.get("mount_point")
 354       print("Error: Not enough room on %s (total: %d blocks, used: %d blocks, "
 355             "reserved: %d blocks, available: %d blocks)" % (
 356                 mount_point, total_blocks, used_blocks, reserved_blocks,
 357                 adjusted_blocks))
 358       return False
 359
 360   if not fs_spans_partition:
 361     mount_point = prop_dict.get("mount_point")
 362     partition_size = int(prop_dict.get("partition_size"))
 363     image_size = os.stat(out_file).st_size
 364     if image_size > partition_size:
 365       print("Error: %s image size of %d is larger than partition size of "
 366             "%d" % (mount_point, image_size, partition_size))
 367       return False
 368     if verity_supported and is_verity_partition:
 369       if 2 * image_size - AdjustPartitionSizeForVerity(image_size) > partition_size:
 370         print "Error: No more room on %s to fit verity data" % mount_point
 371         return False
 372     prop_dict["original_partition_size"] = prop_dict["partition_size"]
 373     prop_dict["partition_size"] = str(image_size)
 374
 375   # create the verified image if this is to be verified
 376   if verity_supported and is_verity_partition:
 377     if not MakeVerityEnabledImage(out_file, prop_dict):
 378       return False
 379
 380   if run_fsck and prop_dict.get("skip_fsck") != "true":
 381     success, unsparse_image = UnsparseImage(out_file, replace=False)
 382     if not success:
 383       return False
 384
 385     # Run e2fsck on the inflated image file
 386     e2fsck_command = ["e2fsck", "-f", "-n", unsparse_image]
 387     (_, exit_code) = RunCommand(e2fsck_command)
 388
 389     os.remove(unsparse_image)
 390
 391   return exit_code == 0
 392
 393
 394 def ImagePropFromGlobalDict(glob_dict, mount_point):
 395   """Build an image property dictionary from the global dictionary.
 396
 397   Args:
 398     glob_dict: the global dictionary from the build system.
 399     mount_point: such as "system", "data" etc.
 400   """
 401   d = {}
 402
 403   if "build.prop" in glob_dict:
 404     bp = glob_dict["build.prop"]
 405     if "ro.build.date.utc" in bp:
 406       d["timestamp"] = bp["ro.build.date.utc"]
 407
 408   def copy_prop(src_p, dest_p):
 409     if src_p in glob_dict:
 410       d[dest_p] = str(glob_dict[src_p])
 411
 412   common_props = (
 413       "extfs_sparse_flag",
 414       "mkyaffs2_extra_flags",
 415       "selinux_fc",
 416       "skip_fsck",
 417       "verity",
 418       "verity_key",
 419       "verity_signer_cmd"
 420       )
 421   for p in common_props:
 422     copy_prop(p, p)
 423
 424   d["mount_point"] = mount_point
 425   if mount_point == "system":
 426     copy_prop("fs_type", "fs_type")
 427     # Copy the generic sysetem fs type first, override with specific one if
 428     # available.
 429     copy_prop("system_fs_type", "fs_type")
 430     copy_prop("system_size", "partition_size")
 431     copy_prop("system_journal_size", "journal_size")
 432     copy_prop("system_verity_block_device", "verity_block_device")
 433     copy_prop("system_root_image", "system_root_image")
 434     copy_prop("ramdisk_dir", "ramdisk_dir")
 435     copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
 436     copy_prop("system_squashfs_compressor", "squashfs_compressor")
 437     copy_prop("system_squashfs_compressor_opt", "squashfs_compressor_opt")
 438   elif mount_point == "data":
 439     # Copy the generic fs type first, override with specific one if available.
 440     copy_prop("fs_type", "fs_type")
 441     copy_prop("userdata_fs_type", "fs_type")
 442     copy_prop("userdata_size", "partition_size")
 443   elif mount_point == "cache":
 444     copy_prop("cache_fs_type", "fs_type")
 445     copy_prop("cache_size", "partition_size")
 446   elif mount_point == "vendor":
 447     copy_prop("vendor_fs_type", "fs_type")
 448     copy_prop("vendor_size", "partition_size")
 449     copy_prop("vendor_journal_size", "journal_size")
 450     copy_prop("vendor_verity_block_device", "verity_block_device")
 451     copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
 452   elif mount_point == "oem":
 453     copy_prop("fs_type", "fs_type")
 454     copy_prop("oem_size", "partition_size")
 455     copy_prop("oem_journal_size", "journal_size")
 456     copy_prop("has_ext4_reserved_blocks", "has_ext4_reserved_blocks")
 457
 458   return d
 459
 460
 461 def LoadGlobalDict(filename):
 462   """Load "name=value" pairs from filename"""
 463   d = {}
 464   f = open(filename)
 465   for line in f:
 466     line = line.strip()
 467     if not line or line.startswith("#"):
 468       continue
 469     k, v = line.split("=", 1)
 470     d[k] = v
 471   f.close()
 472   return d
 473
 474
 475 def main(argv):
 476   if len(argv) != 4:
 477     print __doc__
 478     sys.exit(1)
 479
 480   in_dir = argv[0]
 481   glob_dict_file = argv[1]
 482   out_file = argv[2]
 483   target_out = argv[3]
 484
 485   glob_dict = LoadGlobalDict(glob_dict_file)
 486   if "mount_point" in glob_dict:
 487     # The caller knows the mount point and provides a dictionay needed by
 488     # BuildImage().
 489     image_properties = glob_dict
 490   else:
 491     image_filename = os.path.basename(out_file)
 492     mount_point = ""
 493     if image_filename == "system.img":
 494       mount_point = "system"
 495     elif image_filename == "userdata.img":
 496       mount_point = "data"
 497     elif image_filename == "cache.img":
 498       mount_point = "cache"
 499     elif image_filename == "vendor.img":
 500       mount_point = "vendor"
 501     elif image_filename == "oem.img":
 502       mount_point = "oem"
 503     else:
 504       print >> sys.stderr, "error: unknown image file name ", image_filename
 505       exit(1)
 506
 507     image_properties = ImagePropFromGlobalDict(glob_dict, mount_point)
 508
 509   if not BuildImage(in_dir, image_properties, out_file, target_out):
 510     print >> sys.stderr, "error: failed to build %s from %s" % (out_file,
 511                                                                 in_dir)
 512     exit(1)
 513
 514
 515 if __name__ == '__main__':
 516   main(sys.argv[1:])