cookbooks/screwdriver/recipes/docker-compose.rb

   1 #
   2 # Cookbook Name:: screwdriver
   3 # Recipe:: docker-compose
   4 #
   5 # Copyright 2017, whitestar
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 doc_url = 'https://hub.docker.com/r/screwdrivercd/screwdriver/'
  21
  22 ::Chef::Recipe.send(:include, SSLCert::Helper)
  23
  24 #include_recipe 'platform_utils::kernel_user_namespace'
  25 include_recipe 'docker-grid::compose'
  26
  27 default_executor = {
  28   'plugin' => 'docker',
  29   'docker' => {
  30     'options' => {
  31       'docker' => {
  32         'socketPath' => '/var/run/docker.sock',
  33       },
  34       'launchVersion' => 'stable',
  35     },
  36   },
  37 }
  38
  39 app_dir = node['screwdriver']['docker-compose']['app_dir']
  40 bin_dir = node['screwdriver']['docker-compose']['bin_dir']
  41 config_dir = node['screwdriver']['docker-compose']['config_dir']
  42 data_dir = node['screwdriver']['docker-compose']['data_dir']
  43 etc_dir = node['screwdriver']['docker-compose']['etc_dir']
  44
  45 [
  46   app_dir,
  47   bin_dir,
  48   config_dir,
  49   data_dir,
  50   "#{etc_dir}/nginx",
  51 ].each {|dir|
  52   resources(directory: dir) rescue directory dir do
  53     owner 'root'
  54     group 'root'
  55     mode '0755'
  56     recursive true
  57   end
  58 }
  59
  60 api_config_file = "#{config_dir}/api-local.yaml"
  61 env_file = "#{app_dir}/.env"
  62 config_file = "#{app_dir}/docker-compose.yml"
  63
  64 api_config_local = nil
  65 if File.exist?(api_config_file)
  66   require 'yaml'
  67   api_config_local = YAML.load_file(api_config_file)
  68 end
  69
  70 env_local = nil
  71 if File.exist?(env_file)
  72   env_local = {}
  73   File.open(env_file) do |file|
  74     file.each_line do |line|
  75       env_local[$1] = $2 if line =~ /^([^=]*)=(.*)$/
  76     end
  77   end
  78 end
  79
  80 =begin
  81 config_srvs_local = nil
  82 if File.exist?(config_file)
  83   require 'yaml'
  84   config_srvs_local = YAML.load_file(config_file)['services']
  85 end
  86 =end
  87
  88 # We use plain Hash objects instead of Chef attribute objects for containg secrets (JWT key pair).
  89 override_api_config = node['screwdriver']['api']['config'].to_hash
  90 override_store_config = node['screwdriver']['store']['config'].to_hash
  91 #override_api_config = node.override['screwdriver']['api']['config']      # NG
  92 #override_store_config = node.override['screwdriver']['store']['config']  # NG
  93
  94 config_srvs = node['screwdriver']['docker-compose']['config']['services']
  95 override_config_srvs = node.override['screwdriver']['docker-compose']['config']['services']
  96 force_override_config_srvs = node.force_override['screwdriver']['docker-compose']['config']['services']
  97
  98 # api
  99 api_envs_org = config_srvs['api']['environment']
 100 api_envs = {}
 101 api_vols = config_srvs['api']['volumes'].to_a
 102
 103 api_port = '9001'  # default
 104 api_in_port = api_envs_org['PORT']
 105 ports = config_srvs['api']['ports']
 106 if ports.empty?
 107   override_config_srvs['api']['ports'] = ["#{api_port}:#{api_in_port}"]
 108 else
 109   ports.each {|port|
 110     elms = port.split(':')
 111     api_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == api_in_port
 112   }
 113 end
 114
 115 override_api_config['executor'] = default_executor if override_api_config['executor'].empty?
 116
 117 [
 118   'jwt_private_key_vault_item',
 119   'jwt_public_key_vault_item',
 120   'cookie_password_vault_item',
 121   'password_vault_item',
 122 ].each {|vault_item|
 123   # for backward compatibility.
 124   if node['screwdriver'][vault_item].empty? && !node['screwdriver']['docker-compose'][vault_item].empty?
 125     node.force_override['screwdriver'][vault_item] = node['screwdriver']['docker-compose'][vault_item].to_hash
 126   end
 127 }
 128
 129 jwt_private_key_reset = node['screwdriver']['docker-compose']['jwt_private_key_reset']
 130 jwt_private_key = nil
 131 jwt_public_key  = nil
 132 jwt_private_key_vault_item = node['screwdriver']['jwt_private_key_vault_item']
 133 jwt_public_key_vault_item  = node['screwdriver']['jwt_public_key_vault_item']
 134
 135 if !jwt_private_key_vault_item.empty?
 136   # 1. from Chef Vault (recommended).
 137   jwt_private_key = get_vault_item_value(jwt_private_key_vault_item)
 138   jwt_public_key  = get_vault_item_value(jwt_public_key_vault_item)
 139   log 'JWT key pair has been loaded from Chef Vault.'
 140 else
 141   # 2. from Chef attribute (NOT recommended).
 142   jwt_private_key = api_envs_org['SECRET_JWT_PRIVATE_KEY']
 143   jwt_public_key  = api_envs_org['SECRET_JWT_PUBLIC_KEY']
 144   if jwt_private_key.nil? || jwt_private_key.empty?
 145     if !api_config_local.nil? && !api_config_local['auth']['jwtPrivateKey'].nil? && !jwt_private_key_reset
 146       # 3. preserve it from the local config/api-local.yaml file.
 147       jwt_private_key = api_config_local['auth']['jwtPrivateKey']
 148       jwt_public_key  = api_config_local['auth']['jwtPublicKey']
 149       log 'JWT key pair is preserved from the local config/api-local.yaml file.'
 150     # if !env_local.nil? && !env_local['SECRET_JWT_PRIVATE_KEY'].nil? && !jwt_private_key_reset
 151     #   # 3. preserve it from the local .env file.
 152     #   # Note: Docker env file format does not support backslash escaped string yet.
 153     #   eval "jwt_private_key = %Q(#{env_local['SECRET_JWT_PRIVATE_KEY']})"
 154     #   eval "jwt_public_key  = %Q(#{env_local['SECRET_JWT_PUBLIC_KEY']})"
 155     #   log 'JWT key pair is preserved from the local .env file.'
 156     else
 157       # 4. auto generate.
 158       require 'openssl'
 159       rsa = OpenSSL::PKey::RSA.generate(2048)
 160       jwt_private_key = rsa.export
 161       jwt_public_key  = rsa.public_key.export
 162       log 'JWT key pair has been generated.'
 163     end
 164   end
 165 end
 166
 167 override_api_config['auth']['jwtPrivateKey'] = jwt_private_key
 168 override_api_config['auth']['jwtPublicKey'] = jwt_public_key
 169 # Note: prevent Chef from logging JWT key attribute values. (=> template variables)
 170 # However Docker env file format does not support multi-line value and backslash escaped string yet.
 171 #api_envs['SECRET_JWT_PRIVATE_KEY'] = '${SECRET_JWT_PRIVATE_KEY}'  # Useless
 172 #api_envs['SECRET_JWT_PUBLIC_KEY']  = '${SECRET_JWT_PUBLIC_KEY}'   # Useless
 173 #api_envs['SECRET_JWT_PRIVATE_KEY'] = jwt_private_key  # NG
 174 #api_envs['SECRET_JWT_PUBLIC_KEY']  = jwt_public_key   # NG
 175
 176 cookie_password = nil
 177 cookie_password_vault_item = node['screwdriver']['cookie_password_vault_item']
 178 unless cookie_password_vault_item.empty?
 179   cookie_password = get_vault_item_value(cookie_password_vault_item)
 180   api_envs['SECRET_COOKIE_PASSWORD'] = '${SECRET_COOKIE_PASSWORD}'
 181 end
 182
 183 password = nil
 184 password_vault_item = node['screwdriver']['password_vault_item']
 185 unless password_vault_item.empty?
 186   password = get_vault_item_value(password_vault_item)
 187   api_envs['SECRET_PASSWORD'] = '${SECRET_PASSWORD}'
 188 end
 189
 190 node['screwdriver']['api']['scms_vault_items'].each {|scm, props|
 191   props.each {|prop, vault_item|
 192     unless vault_item.empty?
 193       secret = get_vault_item_value(vault_item)
 194       override_api_config['scms'][scm]['config'][prop] = secret
 195     end
 196   }
 197 }
 198 =begin
 199 # **DEPRECATED!!**
 200 oauth_client_id = nil
 201 oauth_client_id_vault_item = node['screwdriver']['docker-compose']['oauth_client_id_vault_item']
 202 unless oauth_client_id_vault_item.empty?
 203   oauth_client_id = get_vault_item_value(oauth_client_id_vault_item)
 204   api_envs['SECRET_OAUTH_CLIENT_ID'] = '${SECRET_OAUTH_CLIENT_ID}'
 205 end
 206
 207 oauth_client_secret = nil
 208 oauth_client_secret_vault_item = node['screwdriver']['docker-compose']['oauth_client_secret_vault_item']
 209 unless oauth_client_secret_vault_item.empty?
 210   oauth_client_secret = get_vault_item_value(oauth_client_secret_vault_item)
 211   api_envs['SECRET_OAUTH_CLIENT_SECRET'] = '${SECRET_OAUTH_CLIENT_SECRET}'
 212 end
 213
 214 webhook_github_secret = nil
 215 webhook_github_secret_vault_item = node['screwdriver']['docker-compose']['webhook_github_secret_vault_item']
 216 unless webhook_github_secret_vault_item.empty?
 217   webhook_github_secret = get_vault_item_value(webhook_github_secret_vault_item)
 218   api_envs['WEBHOOK_GITHUB_SECRET'] = '${WEBHOOK_GITHUB_SECRET}'
 219 end
 220 =end
 221
 222 db_username = nil
 223 db_username_vault_item = node['screwdriver']['db_username_vault_item']
 224 unless db_username_vault_item.empty?
 225   db_username = get_vault_item_value(db_username_vault_item)
 226   api_envs['DATASTORE_SEQUELIZE_USERNAME'] = '${DB_USERNAME}'
 227 end
 228
 229 db_password = nil
 230 db_password_vault_item = node['screwdriver']['db_password_vault_item']
 231 unless db_password_vault_item.empty?
 232   db_password = get_vault_item_value(db_password_vault_item)
 233   api_envs['DATASTORE_SEQUELIZE_PASSWORD'] = '${DB_PASSWORD}'
 234 end
 235
 236 db_root_password = nil
 237 db_root_password_vault_item = node['screwdriver']['db_root_password_vault_item']
 238 unless db_root_password_vault_item.empty?
 239   db_root_password = get_vault_item_value(db_root_password_vault_item)
 240 end
 241
 242 db_dialect = api_envs_org['DATASTORE_SEQUELIZE_DIALECT']
 243 case db_dialect
 244 when 'sqlite'
 245   api_vols.push("#{data_dir}:/sd-data:rw")
 246   api_envs['DATASTORE_SEQUELIZE_STORAGE'] = '/sd-data/storage.db'
 247 when 'mysql'
 248   override_config_srvs['api']['links'] = ['db']
 249   api_envs['DATASTORE_SEQUELIZE_HOST'] = 'db'
 250 end
 251
 252 # db
 253 if db_dialect != 'sqlite'
 254   #db_envs_org = config_srvs['db']['environment']
 255   db_envs = {}
 256   db_vols = config_srvs['db']['volumes'].to_a
 257
 258   case db_dialect
 259   when 'mysql'
 260     mysql_data_dir = "#{data_dir}/mysql"
 261     resources(directory: mysql_data_dir) rescue directory mysql_data_dir do
 262       owner 'root'
 263       group 'root'
 264       mode '0755'
 265       recursive true
 266     end
 267
 268     db_envs['MYSQL_DATABASE'] = api_envs_org['DATASTORE_SEQUELIZE_DATABASE']
 269     db_envs['MYSQL_USER'] = '${DB_USERNAME}' unless db_username.nil?
 270     db_envs['MYSQL_PASSWORD'] = '${DB_PASSWORD}' unless db_password.nil?
 271     db_envs['MYSQL_ROOT_PASSWORD'] = '${DB_ROOT_PASSWORD}' unless db_root_password.nil?
 272     db_vols.push("#{mysql_data_dir}:/var/lib/mysql:rw")
 273   end
 274 end
 275
 276 # ui
 277 #ui_envs_org = config_srvs['ui']['environment']
 278 ui_envs = {}
 279 ui_vols = config_srvs['ui']['volumes'].to_a
 280
 281 ui_port = '9000'  # default
 282 ui_in_port = '80'
 283 ports = config_srvs['ui']['ports']
 284 if ports.empty?
 285   override_config_srvs['ui']['ports'] = ["#{ui_port}:#{ui_in_port}"]
 286 else
 287   ports.each {|port|
 288     elms = port.split(':')
 289     ui_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == ui_in_port
 290   }
 291 end
 292
 293 # store
 294 store_envs_org = config_srvs['store']['environment']
 295 store_envs = {}
 296 store_vols = config_srvs['store']['volumes'].to_a
 297
 298 store_port = '9002'  # default
 299 store_in_port = store_envs_org['PORT']
 300 ports = config_srvs['store']['ports']
 301 if ports.empty?
 302   override_config_srvs['store']['ports'] = ["#{store_port}:#{store_in_port}"]
 303 else
 304   ports.each {|port|
 305     elms = port.split(':')
 306     store_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == store_in_port
 307   }
 308 end
 309
 310 override_store_config['auth']['jwtPublicKey'] = jwt_public_key
 311 # Note: prevent Chef from logging JWT key attribute value. (=> template variables)
 312 # However Docker env file format does not support multi-line value and backslash escaped string yet.
 313 #store_envs['SECRET_JWT_PUBLIC_KEY'] = '${SECRET_JWT_PUBLIC_KEY}'  # Useless
 314 #store_envs['SECRET_JWT_PUBLIC_KEY']  = jwt_public_key  # NG
 315
 316 api_uri = api_envs_org['URI']
 317 store_uri = store_envs_org['URI']
 318 ui_uri = api_uri.gsub(/:\d+/, ":#{ui_port}")  # based on the API URI.
 319
 320 if node['screwdriver']['with_ssl_cert_cookbook']
 321   cn = node['screwdriver']['ssl_cert']['common_name']
 322   append_server_ssl_cn(cn)
 323   include_recipe 'ssl_cert::server_key_pairs'
 324
 325   server_cert = server_cert_content(cn)
 326   server_key = server_key_content(cn)
 327
 328   api_uri = api_uri.gsub('http://', 'https://')
 329   store_uri = store_uri.gsub('http://', 'https://')
 330
 331   override_api_config['httpd']['tls'] = {}  # for FalseClass by default.
 332   override_api_config['httpd']['tls']['cert'] = server_cert
 333   override_api_config['httpd']['tls']['key'] = server_key
 334   api_envs['IS_HTTPS'] = 'true'
 335
 336   override_store_config['httpd']['tls'] = {}  # for FalseClass by default.
 337   override_store_config['httpd']['tls']['cert'] = server_cert
 338   override_store_config['httpd']['tls']['key'] = server_key
 339
 340   # Note: Screwdriver UI image does not support TLS settings yet.
 341   # https://github.com/screwdriver-cd/screwdriver/issues/377
 342
 343   if node['screwdriver']['ui']['tls_setup_mode'] == 'reverseproxy'
 344     rproxy_in_port = '9000'
 345     ports = config_srvs['reverseproxy']['ports']
 346     if ports.empty?
 347       override_config_srvs['reverseproxy']['ports'] = ["#{ui_port}:#{rproxy_in_port}"]
 348     else
 349       ports.each {|port|
 350         elms = port.split(':')
 351         ui_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == rproxy_in_port
 352       }
 353     end
 354     ui_uri = api_uri.gsub(/:\d+/, ":#{ui_port}")  # based on the API URI.
 355     # do not expose UI service directly.
 356     node.rm('screwdriver', 'docker-compose', 'config', 'services', 'ui', 'ports')
 357
 358     rproxy_vols = config_srvs['reverseproxy']['volumes'].to_a
 359     rproxy_vols.push("#{etc_dir}/nginx/nginx.conf:/etc/nginx/nginx.conf:ro")
 360     # Nginx parent process owner is root.
 361     rproxy_vols.push("#{server_cert_path(cn)}:/root/server.crt:ro")
 362     rproxy_vols.push("#{server_key_path(cn)}:/root/server.key:ro")
 363     # reset vlumes array.
 364     override_config_srvs['reverseproxy']['volumes'] = rproxy_vols
 365
 366     template "#{etc_dir}/nginx/nginx.conf" do
 367       source 'opt/docker-compose/app/screwdriver/etc/nginx/nginx.conf'
 368       owner 'root'
 369       group 'root'
 370       mode '0644'
 371       action :create
 372     end
 373   else
 374     node.rm('screwdriver', 'docker-compose', 'config', 'services', 'reverseproxy')
 375     # TODO: in the future.
 376   end
 377 else
 378   node.rm('screwdriver', 'docker-compose', 'config', 'services', 'reverseproxy')
 379 end
 380
 381 # normalize URIs
 382 api_envs['URI'] = api_uri
 383 api_envs['ECOSYSTEM_STORE'] = store_uri
 384 api_envs['ECOSYSTEM_UI'] = ui_uri
 385
 386 ui_envs['ECOSYSTEM_API'] = api_uri
 387 ui_envs['ECOSYSTEM_STORE'] = store_uri
 388
 389 store_envs['URI'] = store_uri
 390 store_envs['ECOSYSTEM_UI'] = ui_uri
 391
 392 # Common
 393 if node['screwdriver']['docker-compose']['import_ca']
 394   node['screwdriver']['ssl_cert']['ca_names'].each {|ca_name|
 395     append_ca_name(ca_name)
 396     ca_cert_vol = "#{ca_cert_path(ca_name)}:/usr/share/ca-certificates/#{ca_name}.crt:ro"
 397     api_vols.push(ca_cert_vol)
 398     #ui_vols.push(ca_cert_vol)
 399   }
 400   include_recipe 'ssl_cert::ca_certs'
 401
 402   import_ca_script = '/usr/local/bin/screwdriver_import_ca'
 403   template "#{bin_dir}/screwdriver_import_ca" do
 404     source 'opt/docker-compose/app/screwdriver/bin/screwdriver_import_ca'
 405     owner 'root'
 406     group 'root'
 407     mode '0755'
 408     action :create
 409   end
 410   import_ca_script_vol = "#{bin_dir}/screwdriver_import_ca:#{import_ca_script}:ro"
 411   api_vols.push(import_ca_script_vol)
 412   #ui_vols.push(import_ca_script_vol)
 413
 414   api_command = config_srvs['api']['command']
 415   override_config_srvs['api']['command'] \
 416     = "/bin/sh -c \"#{import_ca_script} && #{api_command}\""
 417 end
 418
 419 [
 420   'api',
 421   'store',
 422 ].each {|srv|
 423   local_yaml_file = "#{config_dir}/#{srv}-local.yaml"
 424   srv_vols = nil
 425   srv_config = nil
 426   if srv == 'api'
 427     srv_vols = api_vols
 428     srv_config = override_api_config
 429   elsif srv == 'store'
 430     srv_vols = store_vols
 431     srv_config = override_store_config
 432   end
 433
 434   template local_yaml_file do
 435     source "opt/docker-compose/app/screwdriver/config/#{srv}-local.yaml"
 436     owner 'root'
 437     group 'root'
 438     mode '0600'
 439     sensitive true
 440     # prevent Chef from logging password attribute value.
 441     variables(
 442       config: srv_config
 443     )
 444   end
 445
 446   srv_vols.push("#{local_yaml_file}:/config/local.yaml:ro")
 447 }
 448
 449 # merge environment hash
 450 force_override_config_srvs['api']['environment'] = api_envs unless api_envs.empty?
 451 force_override_config_srvs['ui']['environment'] = ui_envs unless ui_envs.empty?
 452 force_override_config_srvs['store']['environment'] = store_envs unless store_envs.empty?
 453 if db_dialect != 'sqlite'
 454   force_override_config_srvs['db']['environment'] = db_envs unless db_envs.empty?
 455 end
 456 # reset vlumes array.
 457 override_config_srvs['api']['volumes'] = api_vols unless api_vols.empty?
 458 override_config_srvs['ui']['volumes'] = ui_vols unless ui_vols.empty?
 459 override_config_srvs['store']['volumes'] = store_vols unless store_vols.empty?
 460 if db_dialect != 'sqlite'
 461   override_config_srvs['db']['volumes'] = db_vols unless db_vols.empty?
 462 end
 463
 464 template env_file do
 465   source 'opt/docker-compose/app/screwdriver/.env'
 466   owner 'root'
 467   group 'root'
 468   mode '0600'
 469   sensitive true
 470   # prevent Chef from logging password attribute value.
 471   variables(
 472     # secrets
 473     cookie_password: cookie_password,
 474     password: password,
 475     db_username: db_username,
 476     db_password: db_password,
 477     db_root_password: db_root_password,
 478     # **DEPRECATED!!**
 479     # JWT keys setting -> /config/local.yaml
 480     #jwt_private_key: jwt_private_key,
 481     #jwt_public_key: jwt_public_key,
 482     # SCM secrets setting -> /config/local.yaml
 483     #oauth_client_id: oauth_client_id,
 484     #oauth_client_secret: oauth_client_secret,
 485     #webhook_github_secret: webhook_github_secret
 486   )
 487 end
 488
 489 template config_file do
 490   source 'opt/docker-compose/app/screwdriver/docker-compose.yml'
 491   owner 'root'
 492   group 'root'
 493   mode '0644'
 494 end
 495
 496 log 'screwdriver docker-compose post install message' do
 497   message <<-"EOM"
 498 Note: You must execute the following command manually.
 499     See #{doc_url}
 500     * Start:
 501       $ cd #{app_dir}
 502       $ sudo docker-compose up -d
 503     * Stop
 504       $ sudo docker-compose down
 505 EOM
 506 end