#!/usr/bin/ruby
+#
+# Author: Tatsuki Sugiura <sugi@nemui.org>
+# Copyright: Apprits, Inc.
+# Lisence: MIT
+#
+#
require 'pp'
require 'shellwords'
require 'tmpdir'
end
class ImageCreator
- attr_accessor :name, :dirs, :src_host, :img_path_base
+ attr_accessor :name, :dirs, :src_host, :img_path_base, :run_cmds
MiB = 1024 ** 2
GiB = 1024 ** 3
- def initialize(name, dirs, src_host: nil)
+ def initialize(name, dirs, src_host: nil, run_cmds: nil)
@name = name
@dirs = dirs
@src_host = src_host || name
+ @run_cmds = run_cmds
@img_path_base = "#{name}_#{Time.now.strftime '%FT%T%z'}"
end
begin
system("mount", dev, mount_point) or raise "Failed to mount file system #{dev} on #{mount_point}"
puts "Copying #{src_host}:#{di.srcpath} to #{dev}..."
- system("rsync", "-azHSAX", "--numeric-ids", "--info=progress2", "#{src_host}:#{di.srcpath}/", "#{mount_point}/", *((["--exclude"] * di.exclude.size).zip(di.exclude).flatten)) or raise "rsync fails"
+ unless system("rsync", "-azHSAX", "--numeric-ids", "--info=progress2", "#{src_host}:#{di.srcpath}/", "#{mount_point}/", *((["--exclude"] * di.exclude.size).zip(di.exclude).flatten))
+ warn "rsync exit with error, file transfer may not be completed."
+ end
ensure
system("umount", mount_point)
File.directory?(mount_point) and
end
end
+ def prepare_awsenv
+ puts "Prepareing AWS cloud boot environment..."
+ with_rootfs do |dir, devices|
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, *%w(sudo dpkg --remove td-agent smartmontools snmpd lm-sensors arrayprobe megacli megacli-check-change megaclisas megaclisas-status fancontrol))
+
+ pkgs = %w(apt-transport-https locales-all ntp ntpdate rsync python-apt git subversion less lv cloud-init tmux screen irqbalance)
+ if File.read("#{dir}/etc/debian_version").to_f >= 9.0
+ pkgs << 'cloud-guest-utils'
+ end
+
+ system("chroot", dir, "apt-get", "-qy", "update")
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "--allow-unauthenticated", "install", *pkgs)
+
+ if File.exists? "#{dir}//var/spool/nullmailer/queue"
+ puts "Remove nullmailer queue"
+ system("chroot", dir, "find", "/var/spool/nullmailer/queue", "-type", "f", "-name", '[0-9]*.[0-9]*', "-delete")
+ end
+
+ unless system("chroot", dir, *%w(grep -rq cdn-aws.deb.debian.org /etc/apt/sources.list /etc/apt/sources.list.d))
+ puts "Rewrite apt sources list for AWS CDN"
+ rel = `chroot #{dir} lsb_release -c -s`.chomp
+ s = [
+ "deb http://cdn-aws.deb.debian.org/debian #{rel} main main contrib non-free",
+ "deb http://cdn-aws.deb.debian.org/debian #{rel}-updates main main contrib non-free",
+ ]
+ if File.exists? "#{dir}/etc/apt/sources.list"
+ s << File.read("#{dir}/etc/apt/sources.list")
+ end
+ File.write "#{dir}/etc/apt/sources.list", s.join("\n")
+ end
+
+ unless File.read("#{dir}/etc/ntp.conf").include? "169.254.169.123"
+ puts "Rewrite ntp.conf"
+ c = File.read("#{dir}/etc/ntp.conf")
+ c.gsub!(/^(pool|server) /) { "##{$1}" }
+ c << "\nserver 169.254.169.123 prefer iburst\n"
+ File.write "#{dir}/etc/ntp.conf", c
+ end
+
+ File.write "#{dir}/etc/cloud/cloud.cfg.d/50_keep_apt_source.cfg", "apt_preserve_sources_list: true\n"
+ File.write "#{dir}/etc/cloud/cloud.cfg.d/50_keep_ssh_key.cfg", "ssh_deletekeys: false\n"
+ end
+ end
+
def fix_boot
puts "Fixing boot environments..."
+ with_rootfs do |dir, devices|
+ puts "Override grub with host version..."
+ root_dev = "/dev/#{devices.first[/loop\d+/]}"
+ rootfs_uuid = dirs.find { |d| d.path == '/'}.fs_uuid
+ puts "New rootfs UUID=#{rootfs_uuid}"
+
+ system "rm", "-f", "#{dir}/etc/systemd/system/udev.service", "#{dir}/etc/systemd/system/systemd-udevd.service", "#{dir}/etc/udev/rules.d/70-persistent-net.rules"
+
+ puts "Rewrite fstab..."
+ File.open "#{dir}/etc/fstab", "w" do |f|
+ dirs.each_with_index do |di, idx|
+ f << %W(UUID=#{di.fs_uuid} #{di.path} ext4 defaults,noatime 0 #{di.path == '/' ? 1 : 2}).join("\t")
+ f << "\n"
+ end
+ end
+
+ system("chroot", dir, "apt-get", "-qy", "update")
+ if File.read("#{dir}/etc/debian_version").to_f >= 9.0
+ # Note: 2019-10-08 時点で Debian9 のカーネルバージョンに対応していないので、エラー回避のために既存のカーネルを全て削除し、強制的に jessie のカーネルをイントールする
+ system("rm", "-f", "#{dir}/var/lib/dpkg/info/linux-image-#{`uname -r`.chomp}.prerm")
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "remove", "--purge", "-y", "linux-image-*")
+ system("wget", "-O", "#{dir}/tmp/linux.deb", "http://security-cdn.debian.org/debian-security/pool/updates/main/l/linux/linux-image-3.16.0-10-amd64_3.16.81-1_amd64.deb") or raise "Failed to get jessie kernel"
+ system("chroot", dir, "dpkg", "-i", "/tmp/linux.deb")
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "install", "-f", "-y") or raise "Failed to install jessie kernel"
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "-y", "install", "isc-dhcp-client")
+ else
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "--allow-unauthenticated", "install", "linux-image-amd64", "isc-dhcp-client")
+ end
+
+ puts "Update grub..."
+ if File.exists? "#{dir}/var/lib/dpkg/info/grub-efi-amd64.list"
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt", "remove", "--purge", "-y", "grub-efi-amd64") or raise "Failed to purge grub-efi"
+ end
+ if !File.exists?("#{dir}/var/lib/dpkg/info/grub-pc.list") || !(File.exists?("#{dir}/usr/sbin/grub-bios-setup") || File.exists?("#{dir}/usr/sbin/grub-setup"))
+ system("chroot", dir, "apt-get", "-qy", "update") or raise "Failed to install grub-pc"
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "install", "grub-pc")
+ end
+ File.open "#{dir}/boot/grub/device.map", "w" do |f|
+ f.puts "(hd0)\t#{root_dev}"
+ end
+ system("chroot", dir, "grub-mkconfig", "-o", "/boot/grub/grub.cfg") or raise "grub-mkconfig fails."
+ system(*%W(grub-install --no-floppy --grub-mkdevicemap=#{dir}/boot/grub/device.map --root-directory=#{dir} #{root_dev})) or raise "grub-install failed."
+
+ unless Array(run_cmds).empty?
+ Array(run_cmds).each do |cmd|
+ system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, *Array(cmd)) or raise "Failed to execute command (#{cmd}): #{$!}"
+ end
+ end
+
+ cfg = File.read "#{dir}/boot/grub/grub.cfg"
+ cfg.gsub! %r{mapper/loop\d+p}, "sda"
+ File.write "#{dir}/boot/grub/grub.cfg", cfg
+ end
+ end
+
+ def with_rootfs(&block)
Dir.mktmpdir("ci-#{$$}-#{name}") do |dir|
with_loopdev do |devices|
- puts "Override grub with host version..."
- root_dev = "/dev/#{devices.first[/loop\d+/]}"
- rootfs_uuid = dirs.find { |d| d.path == '/'}.fs_uuid
- puts "New rootfs UUID=#{rootfs_uuid}"
begin
system("mount", devices.first, dir) or raise "Failed to mount #{devices.first} to #{dir}"
system("mount", "--bind", "/dev", "#{dir}/dev") or raise "Failed to mount /dev to #{dir}/dev"
system("mount", di.device, "#{dir}#{di.path}") or raise "Failed to mount #{di.device} to #{dir}#{path}"
end
- system "rm", "-f", "#{dir}/etc/systemd/system/udev.service", "#{dir}/etc/systemd/system/systemd-udevd.service"
+ yield(dir, devices)
- puts "Rewrite fstab..."
- File.open "#{dir}/etc/fstab", "w" do |f|
- dirs.each_with_index do |di, idx|
- f << %W(UUID=#{di.fs_uuid} #{di.path} ext4 defaults,noatime 0 #{di.path == '/' ? 1 : 2}).join("\t")
- f << "\n"
- end
- end
-
- unless File.exists? "#{dir}/vmlinuz"
- system("chroot", dir, "apt-get", "-qy", "update")
- system("chroot", dir, "apt-get", "-y", "install", "linux-image-amd64")
- end
-
- puts "Update grub..."
- unless File.exists? "#{dir}/usr/sbin/grub-mkconfig"
- system("chroot", dir, "apt-get", "-qy", "update") or raise "Failed to install grub-pc"
- system({'DEBIAN_FRONTEND' => 'noninteractive'}, "chroot", dir, "apt-get", "-y", "install", "grub-pc")
- end
- File.open "#{dir}/boot/grub/device.map", "w" do |f|
- f.puts "(hd0)\t#{root_dev}"
- end
- system("chroot", dir, "grub-mkconfig", "-o", "/boot/grub/grub.cfg") or raise "grub-mkconfig fails."
- system(*%W(grub-install --no-floppy --grub-mkdevicemap=#{dir}/boot/grub/device.map --root-directory=#{dir} #{root_dev})) or raise "grub-install failed."
ensure
system("umount", "#{dir}/dev")
system("umount", "#{dir}/proc")
"Description" => dir.path == '/' ? 'root' : dir.path[1..-1].tr('/', '-'),
"Format" => "raw",
"UserBucket" => {
- "S3Bucket" => "Change-to-your-buket-name",
- "S3Key" => "/src-disks/#{img_path_base}_#{idx}.img"
+ "S3Bucket" => ENV.fetch("S3_BUCKET", "osdn-base-images"),
+ "S3Key" => "#{ENV.fetch("S3_KEY_PREFIX", "src-disks/")}#{img_path_base}_#{idx}.img"
}
})
end
File.write "#{img_path_base}.json", JSON.pretty_generate(jdef)
end
-
def run
create_disk
create_fs
sync_dirs
+ prepare_awsenv
fix_boot
write_json
puts "Image creation has been complated (#{name})"
end
if $0 == __FILE__
+ require 'optparse'
+
+ opts = ARGV.getopts('l:', 'limit:', 'skip:')
+ limit_pat = opts['limit'] || opts['l']
+ limit_pat and
+ limit_pat = Regexp.new(limit_pat)
+
list = YAML.load_file(ARGV[0] || 'image-list.yml')
list.each do |imgdef|
name = nil
dirs = []
+ opts = {}
if imgdef.kind_of?(Hash)
name = imgdef['name']
(imgdef['dirs'] || {}).each do |path, opts|
opts.kind_of?(Hash) or opts = {size: opts}
dirs << SyncDirDef.new({path: path}.merge(opts.keys.map(&:to_sym).zip(opts.values).to_h))
end
+ imgdef.keys.each do |k|
+ next if %w[dirs name].member?(k)
+ opts[k.to_sym] = imgdef[k]
+ end
else
name = imgdef
end
+ if limit_pat
+ limit_pat.match?(name) or next
+ end
dirs.empty? and dirs << SyncDirDef.new
- ImageCreator.new(name, dirs).run
+ ImageCreator.new(name, dirs, **opts).run
end
end
OSDN Git Service
