create-image

   1 #!/usr/bin/ruby
   2 #
   3 # Author: Tatsuki Sugiura <sugi@nemui.org>
   4 # Copyright: Apprits, Inc.
   5 # Lisence: MIT
   6 #
   7 #
   8 require 'pp'
   9 require 'shellwords'
  10 require 'tmpdir'
  11 require 'yaml'
  12 require 'json'
  13
  14 class SyncDirDef
  15   DEFAULT_EXCLUDE = %w[/proc/* /sys/* /dev/mqueue /dev/hugepages /run/* /var/lib/os-prober/mount /swap /dev/shm/* /var/lib/lxcfs/*]
  16   attr_accessor :path, :size, :exclude, :size, :srcpath, :fs_features, :device, :fs_uuid
  17   def initialize(path: '/', size: 8, exclude: DEFAULT_EXCLUDE, srcpath: nil, fs_features: nil)
  18     @path = path
  19     @size = size.to_f
  20     @exclude = (DEFAULT_EXCLUDE + [*exclude]).uniq
  21     @srcpath = srcpath || path
  22     @fs_features = fs_features
  23     @device = nil
  24     @fs_uuid = nil
  25   end
  26 end
  27
  28 class ImageCreator
  29   attr_accessor :name, :dirs, :src_host, :img_path_base, :run_cmds
  30   MiB = 1024 ** 2
  31   GiB = 1024 ** 3
  32
  33   def initialize(name, dirs, src_host: nil, run_cmds: nil)
  34     @name = name
  35     @dirs = dirs
  36     @src_host = src_host || name
  37     @run_cmds = run_cmds
  38     @img_path_base = "#{name}_#{Time.now.strftime '%FT%T%z'}"
  39   end
  40
  41   def imgpath(idx)
  42     "#{img_path_base}_#{idx}.img"
  43   end
  44
  45   def create_disk
  46     dirs.each_with_index do |di, idx|
  47       _create_disk imgpath(idx), di, idx != 0
  48     end
  49   end
  50
  51   def _create_disk path, di, use_gpt = false
  52     size_gb = di.size
  53     raise "Disk image #{path} is already exists!" if File.exists? path
  54     puts "Creating disk image #{path} (#{'%5.2f' % size_gb.to_f} GiB)..."
  55     File.open(path, "w") do |f|
  56       f.truncate(size_gb * GiB)
  57     end
  58     system("parted", "-s", path, "mklabel", use_gpt ? 'gpt' : 'msdos') or raise "Failed to create partition label"
  59     system("parted", "-s", path, "mkpart", "primary", "1MiB", "#{size_gb * 1024 - 1}MiB") or raise "Failed to create partition"
  60     if !use_gpt
  61       system("parted", "-s", path, "set", "1", "boot", "on") or raise "Failed to set bios boot partition"
  62     end
  63     puts "Image partition has been created."
  64   end
  65
  66   def with_loopdev &block
  67     begin
  68       devices = []
  69       dirs.each_with_index do |di, idx|
  70         system("kpartx", "-as", imgpath(idx)) or raise "Failed to map loop device"
  71         di.device = "/dev/mapper/" + `kpartx -l #{Shellwords.escape imgpath(idx)}`.split("\n").first[/loop\d+p\d+/]
  72         devices << di.device
  73       end
  74       yield devices
  75     ensure
  76       dirs.each_with_index do |di, idx|
  77         system "kpartx", "-d", imgpath(idx), err: "/dev/null"
  78         di.device = nil
  79       end
  80     end
  81   end
  82
  83   def create_fs
  84     with_loopdev do |devices|
  85       dirs.each_with_index do |di, index|
  86         dev = di.device
  87         puts "Creating filesystem on #{dev}..."
  88         cmd = %w(mkfs.ext4 -q)
  89         di = dirs[index]
  90         if di.fs_features
  91           cmd << '-O' << di.fs_features
  92         end
  93         cmd << dev
  94         system(*cmd) or raise "Failed to create file system on #{dev}"
  95         system "e2label", dev, di.path == '/' ? 'ROOT' : di.path[1..-1].tr('/', '-')
  96         di.fs_uuid = `blkid -o value -s UUID #{di.device}`.chomp("\n")
  97       end
  98     end
  99   end
 100
 101   def sync_dirs
 102     with_loopdev do |devices|
 103       devices.each_with_index do |dev, idx|
 104         di = dirs[idx]
 105         mount_point = "/mnt/ci-#{$$}-#{name}-#{idx}"
 106         system("mkdir", "-p", mount_point)
 107         begin
 108           system("mount", dev, mount_point) or raise "Failed to mount file system #{dev} on #{mount_point}"
 109           puts "Copying #{src_host}:#{di.srcpath} to #{dev}..."
 110           unless system("rsync", "-azHSAX", "--numeric-ids", "--info=progress2", "#{src_host}:#{di.srcpath}/", "#{mount_point}/", *((["--exclude"] * di.exclude.size).zip(di.exclude).flatten))
 111             warn "rsync exit with error, file transfer may not be completed."
 112           end
 113         ensure
 114           system("umount", mount_point)
 115           File.directory?(mount_point) and
 116             Dir.rmdir mount_point
 117         end
 118       end
 119     end
 120   end
 121
 122   def prepare_awsenv
 123     puts "Prepareing AWS cloud boot environment..."
 124     with_rootfs do |dir, devices|
 125       system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, *%w(sudo dpkg --remove td-agent smartmontools snmpd lm-sensors arrayprobe megacli megacli-check-change megaclisas megaclisas-status fancontrol))
 126
 127       pkgs = %w(apt-transport-https locales-all ntp ntpdate rsync python-apt git subversion less lv cloud-init tmux screen irqbalance)
 128       if File.read("#{dir}/etc/debian_version").to_f >= 9.0
 129         pkgs << 'cloud-guest-utils'
 130       end
 131
 132       system("chroot", dir, "apt-get", "-qy", "update")
 133       system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "--allow-unauthenticated", "install", *pkgs)
 134
 135       if File.exists? "#{dir}//var/spool/nullmailer/queue"
 136         puts "Remove nullmailer queue"
 137         system("chroot", dir, "find", "/var/spool/nullmailer/queue", "-type", "f", "-name", '[0-9]*.[0-9]*', "-delete")
 138       end
 139
 140       unless system("chroot", dir, *%w(grep -rq cdn-aws.deb.debian.org /etc/apt/sources.list /etc/apt/sources.list.d))
 141         puts "Rewrite apt sources list for AWS CDN"
 142         rel = `chroot #{dir} lsb_release -c -s`.chomp
 143         s = [
 144           "deb http://cdn-aws.deb.debian.org/debian #{rel} main main contrib non-free",
 145           "deb http://cdn-aws.deb.debian.org/debian #{rel}-updates main main contrib non-free",
 146         ]
 147         if File.exists? "#{dir}/etc/apt/sources.list"
 148           s << File.read("#{dir}/etc/apt/sources.list")
 149         end
 150         File.write "#{dir}/etc/apt/sources.list", s.join("\n")
 151       end
 152
 153       unless File.read("#{dir}/etc/ntp.conf").include? "169.254.169.123"
 154         puts "Rewrite ntp.conf"
 155         c = File.read("#{dir}/etc/ntp.conf")
 156         c.gsub!(/^(pool|server) /) { "##{$1}" }
 157         c << "\nserver 169.254.169.123 prefer iburst\n"
 158         File.write "#{dir}/etc/ntp.conf", c
 159       end
 160
 161       File.write "#{dir}/etc/cloud/cloud.cfg.d/50_keep_apt_source.cfg", "apt_preserve_sources_list: true\n"
 162       File.write "#{dir}/etc/cloud/cloud.cfg.d/50_keep_ssh_key.cfg", "ssh_deletekeys: false\n"
 163     end
 164   end
 165
 166   def fix_boot
 167     puts "Fixing boot environments..."
 168     with_rootfs do |dir, devices|
 169       puts "Override grub with host version..."
 170       root_dev = "/dev/#{devices.first[/loop\d+/]}"
 171       rootfs_uuid = dirs.find { |d| d.path == '/'}.fs_uuid
 172       puts "New rootfs UUID=#{rootfs_uuid}"
 173
 174       system "rm", "-f", "#{dir}/etc/systemd/system/udev.service", "#{dir}/etc/systemd/system/systemd-udevd.service", "#{dir}/etc/udev/rules.d/70-persistent-net.rules"
 175
 176       puts "Rewrite fstab..."
 177       File.open "#{dir}/etc/fstab", "w" do |f|
 178         dirs.each_with_index do |di, idx|
 179           f << %W(UUID=#{di.fs_uuid} #{di.path} ext4 defaults,noatime 0 #{di.path == '/' ? 1 : 2}).join("\t")
 180           f << "\n"
 181         end
 182       end
 183
 184       system("chroot", dir, "apt-get", "-qy", "update")
 185       if File.read("#{dir}/etc/debian_version").to_f >= 9.0
 186         # Note: 2019-10-08 時点で Debian9 のカーネルバージョンに対応していないので、エラー回避のために既存のカーネルを全て削除し、強制的に jessie のカーネルをイントールする
 187         system("rm", "-f", "#{dir}/var/lib/dpkg/info/linux-image-#{`uname -r`.chomp}.prerm")
 188         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "remove", "--purge", "-y", "linux-image-*")
 189         system("wget", "-O", "#{dir}/tmp/linux.deb", "http://security-cdn.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb") or raise "Failed to get jessie kernel"
 190         system("chroot", dir, "dpkg", "-i", "/tmp/linux.deb")
 191         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "install", "-f", "-y") or raise "Failed to install jessie kernel"
 192         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "-y", "install", "isc-dhcp-client")
 193       else
 194         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "--allow-unauthenticated", "install", "linux-image-amd64", "isc-dhcp-client")
 195       end
 196
 197       puts "Update grub..."
 198       if File.exists? "#{dir}/var/lib/dpkg/info/grub-efi-amd64.list"
 199         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "remove", "--purge", "-y", "grub-efi-amd64") or raise "Failed to purge grub-efi"
 200       end
 201       if !File.exists?("#{dir}/var/lib/dpkg/info/grub-pc.list") || !(File.exists?("#{dir}/usr/sbin/grub-bios-setup") || File.exists?("#{dir}/usr/sbin/grub-setup"))
 202         system("chroot", dir, "apt-get", "-qy", "update") or raise "Failed to install grub-pc"
 203         system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "install", "grub-pc")
 204       end
 205       File.open "#{dir}/boot/grub/device.map", "w" do |f|
 206         f.puts "(hd0)\t#{root_dev}"
 207       end
 208       system("chroot", dir, "grub-mkconfig", "-o", "/boot/grub/grub.cfg") or raise "grub-mkconfig fails."
 209       system(*%W(grub-install --no-floppy --grub-mkdevicemap=#{dir}/boot/grub/device.map --root-directory=#{dir} #{root_dev})) or raise "grub-install failed."
 210
 211       unless Array(run_cmds).empty?
 212         Array(run_cmds).each do |cmd|
 213           system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, *Array(cmd)) or raise "Failed to execute command (#{cmd}): #{$!}"
 214         end
 215       end
 216
 217       cfg = File.read "#{dir}/boot/grub/grub.cfg"
 218       cfg.gsub! %r{mapper/loop\d+p}, "sda"
 219       File.write "#{dir}/boot/grub/grub.cfg", cfg
 220     end
 221   end
 222
 223   def with_rootfs(&block)
 224     Dir.mktmpdir("ci-#{$$}-#{name}") do |dir|
 225       with_loopdev do |devices|
 226         begin
 227           system("mount", devices.first, dir) or raise "Failed to mount #{devices.first} to #{dir}"
 228           system("mount", "--bind", "/dev", "#{dir}/dev") or raise "Failed to mount /dev to #{dir}/dev"
 229           system("mount", "--bind", "/proc", "#{dir}/proc") or raise "Failed to mount /proc to #{dir}/proc"
 230
 231           dirs[1..-1].each_with_index do |di, idx|
 232             system "mkdir", "-p", "#{dir}#{di.path}"
 233             system("mount", di.device, "#{dir}#{di.path}") or raise "Failed to mount #{di.device} to #{dir}#{path}"
 234           end
 235
 236           yield(dir, devices)
 237
 238         ensure
 239           system("umount", "#{dir}/dev")
 240           system("umount", "#{dir}/proc")
 241           dirs.reverse[0..-2].each do |di, idx|
 242             system("umount", "#{dir}#{di.path}")
 243           end
 244           system("umount", dir)
 245         end
 246       end
 247     end
 248   end
 249
 250   def write_json
 251     jdef = []
 252     dirs.each_with_index do |dir, idx|
 253       jdef.push({
 254         "Description" => dir.path == '/' ? 'root' : dir.path[1..-1].tr('/', '-'),
 255         "Format" => "raw",
 256         "UserBucket" => {
 257           "S3Bucket" => ENV.fetch("S3_BUCKET", "osdn-base-images"),
 258           "S3Key" => "#{ENV.fetch("S3_KEY_PREFIX", "src-disks/")}#{img_path_base}_#{idx}.img"
 259         }
 260       })
 261     end
 262     File.write "#{img_path_base}.json", JSON.pretty_generate(jdef)
 263   end
 264
 265   def run
 266     create_disk
 267     create_fs
 268     sync_dirs
 269     prepare_awsenv
 270     fix_boot
 271     write_json
 272     puts "Image creation has been complated (#{name})"
 273   end
 274 end
 275
 276 if $0 == __FILE__
 277   require 'optparse'
 278
 279   opts = ARGV.getopts('l:', 'limit:', 'skip:')
 280   limit_pat = opts['limit'] || opts['l']
 281   limit_pat and
 282     limit_pat = Regexp.new(limit_pat)
 283
 284   list = YAML.load_file(ARGV[0] || 'image-list.yml')
 285   list.each do |imgdef|
 286     name = nil
 287     dirs = []
 288     opts = {}
 289     if imgdef.kind_of?(Hash)
 290       name = imgdef['name']
 291       (imgdef['dirs'] || {}).each do |path, opts|
 292         opts.kind_of?(Hash) or opts = {size: opts}
 293         dirs << SyncDirDef.new({path: path}.merge(opts.keys.map(&:to_sym).zip(opts.values).to_h))
 294       end
 295       imgdef.keys.each do |k|
 296         next if %w[dirs name].member?(k)
 297         opts[k.to_sym] = imgdef[k]
 298       end
 299     else
 300       name = imgdef
 301     end
 302     if limit_pat
 303       limit_pat.match?(name) or next
 304     end
 305     dirs.empty? and dirs << SyncDirDef.new
 306     ImageCreator.new(name, dirs, **opts).run
 307   end
 308 end