cookbooks/screwdriver/recipes/docker-compose.rb

   1 #
   2 # Cookbook Name:: screwdriver
   3 # Recipe:: docker-compose
   4 #
   5 # Copyright 2017, whitestar
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #     http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 doc_url = 'https://hub.docker.com/r/screwdrivercd/screwdriver/'
  21
  22 ::Chef::Recipe.send(:include, SSLCert::Helper)
  23
  24 #include_recipe 'platform_utils::kernel_user_namespace'
  25 include_recipe 'docker-grid::compose'
  26
  27 app_dir = node['screwdriver']['docker-compose']['app_dir']
  28 bin_dir = node['screwdriver']['docker-compose']['bin_dir']
  29 config_dir = node['screwdriver']['docker-compose']['config_dir']
  30 data_dir = node['screwdriver']['docker-compose']['data_dir']
  31 etc_dir = node['screwdriver']['docker-compose']['etc_dir']
  32
  33 [
  34   app_dir,
  35   bin_dir,
  36   config_dir,
  37   data_dir,
  38   "#{etc_dir}/nginx",
  39 ].each {|dir|
  40   resources(directory: dir) rescue directory dir do
  41     owner 'root'
  42     group 'root'
  43     mode '0755'
  44     recursive true
  45   end
  46 }
  47
  48 api_config_file = "#{config_dir}/api-local.yaml"
  49 env_file = "#{app_dir}/.env"
  50 config_file = "#{app_dir}/docker-compose.yml"
  51
  52 api_config_local = nil
  53 if File.exist?(api_config_file)
  54   require 'yaml'
  55   api_config_local = YAML.load_file(api_config_file)
  56 end
  57
  58 env_local = nil
  59 if File.exist?(env_file)
  60   env_local = {}
  61   File.open(env_file) do |file|
  62     file.each_line do |line|
  63       env_local[$1] = $2 if line =~ /^([^=]*)=(.*)$/
  64     end
  65   end
  66 end
  67
  68 =begin
  69 config_srvs_local = nil
  70 if File.exist?(config_file)
  71   require 'yaml'
  72   config_srvs_local = YAML.load_file(config_file)['services']
  73 end
  74 =end
  75
  76 # We use plain Hash objects instead of Chef attribute objects for containg secrets (JWT key pair).
  77 override_api_config = node['screwdriver']['api']['config'].to_hash
  78 override_store_config = node['screwdriver']['store']['config'].to_hash
  79 #override_api_config = node.override['screwdriver']['api']['config']      # NG
  80 #override_store_config = node.override['screwdriver']['store']['config']  # NG
  81
  82 config_srvs = node['screwdriver']['docker-compose']['config']['services']
  83 override_config_srvs = node.override['screwdriver']['docker-compose']['config']['services']
  84 force_override_config_srvs = node.force_override['screwdriver']['docker-compose']['config']['services']
  85
  86 # api
  87 api_envs_org = config_srvs['api']['environment']
  88 api_envs = {}
  89 api_vols = config_srvs['api']['volumes'].to_a
  90
  91 api_port = '9001'  # default
  92 api_in_port = api_envs_org['PORT']
  93 ports = config_srvs['api']['ports']
  94 if ports.empty?
  95   override_config_srvs['api']['ports'] = ["#{api_port}:#{api_in_port}"]
  96 else
  97   ports.each {|port|
  98     elms = port.split(':')
  99     api_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == api_in_port
 100   }
 101 end
 102 api_vols.push("#{data_dir}:/sd-data:rw")
 103
 104 jwt_private_key_reset = node['screwdriver']['docker-compose']['jwt_private_key_reset']
 105 jwt_private_key = nil
 106 jwt_public_key  = nil
 107 jwt_private_key_vault_item = node['screwdriver']['docker-compose']['jwt_private_key_vault_item']
 108 jwt_public_key_vault_item  = node['screwdriver']['docker-compose']['jwt_public_key_vault_item']
 109
 110 if !jwt_private_key_vault_item.empty?
 111   # 1. from Chef Vault (recommended).
 112   jwt_private_key = get_vault_item_value(jwt_private_key_vault_item)
 113   jwt_public_key  = get_vault_item_value(jwt_public_key_vault_item)
 114   log 'JWT key pair has been loaded from Chef Vault.'
 115 else
 116   # 2. from Chef attribute (NOT recommended).
 117   jwt_private_key = api_envs_org['SECRET_JWT_PRIVATE_KEY']
 118   jwt_public_key  = api_envs_org['SECRET_JWT_PUBLIC_KEY']
 119   if jwt_private_key.nil? || jwt_private_key.empty?
 120     if !api_config_local.nil? && !api_config_local['auth']['jwtPrivateKey'].nil? && !jwt_private_key_reset
 121       # 3. preserve it from the local config/api-local.yaml file.
 122       jwt_private_key = api_config_local['auth']['jwtPrivateKey']
 123       jwt_public_key  = api_config_local['auth']['jwtPublicKey']
 124       log 'JWT key pair is preserved from the local config/api-local.yaml file.'
 125     # if !env_local.nil? && !env_local['SECRET_JWT_PRIVATE_KEY'].nil? && !jwt_private_key_reset
 126     #   # 3. preserve it from the local .env file.
 127     #   # Note: Docker env file format does not support backslash escaped string yet.
 128     #   eval "jwt_private_key = %Q(#{env_local['SECRET_JWT_PRIVATE_KEY']})"
 129     #   eval "jwt_public_key  = %Q(#{env_local['SECRET_JWT_PUBLIC_KEY']})"
 130     #   log 'JWT key pair is preserved from the local .env file.'
 131     else
 132       # 4. auto generate.
 133       require 'openssl'
 134       rsa = OpenSSL::PKey::RSA.generate(2048)
 135       jwt_private_key = rsa.export
 136       jwt_public_key  = rsa.public_key.export
 137       log 'JWT key pair has been generated.'
 138     end
 139   end
 140 end
 141
 142 override_api_config['auth']['jwtPrivateKey'] = jwt_private_key
 143 override_api_config['auth']['jwtPublicKey'] = jwt_public_key
 144 # Note: prevent Chef from logging JWT key attribute values. (=> template variables)
 145 # However Docker env file format does not support multi-line value and backslash escaped string yet.
 146 #api_envs['SECRET_JWT_PRIVATE_KEY'] = '${SECRET_JWT_PRIVATE_KEY}'  # Useless
 147 #api_envs['SECRET_JWT_PUBLIC_KEY']  = '${SECRET_JWT_PUBLIC_KEY}'   # Useless
 148 #api_envs['SECRET_JWT_PRIVATE_KEY'] = jwt_private_key  # NG
 149 #api_envs['SECRET_JWT_PUBLIC_KEY']  = jwt_public_key   # NG
 150
 151 cookie_password = nil
 152 cookie_password_vault_item = node['screwdriver']['docker-compose']['cookie_password_vault_item']
 153 unless cookie_password_vault_item.empty?
 154   cookie_password = get_vault_item_value(cookie_password_vault_item)
 155   api_envs['SECRET_COOKIE_PASSWORD'] = '${SECRET_COOKIE_PASSWORD}'
 156 end
 157
 158 password = nil
 159 password_vault_item = node['screwdriver']['docker-compose']['password_vault_item']
 160 unless password_vault_item.empty?
 161   password = get_vault_item_value(password_vault_item)
 162   api_envs['SECRET_PASSWORD'] = '${SECRET_PASSWORD}'
 163 end
 164
 165 oauth_client_id = nil
 166 oauth_client_id_vault_item = node['screwdriver']['docker-compose']['oauth_client_id_vault_item']
 167 unless oauth_client_id_vault_item.empty?
 168   oauth_client_id = get_vault_item_value(oauth_client_id_vault_item)
 169   api_envs['SECRET_OAUTH_CLIENT_ID'] = '${SECRET_OAUTH_CLIENT_ID}'
 170 end
 171
 172 oauth_client_secret = nil
 173 oauth_client_secret_vault_item = node['screwdriver']['docker-compose']['oauth_client_secret_vault_item']
 174 unless oauth_client_secret_vault_item.empty?
 175   oauth_client_secret = get_vault_item_value(oauth_client_secret_vault_item)
 176   api_envs['SECRET_OAUTH_CLIENT_SECRET'] = '${SECRET_OAUTH_CLIENT_SECRET}'
 177 end
 178
 179 webhook_github_secret = nil
 180 webhook_github_secret_vault_item = node['screwdriver']['docker-compose']['webhook_github_secret_vault_item']
 181 unless webhook_github_secret_vault_item.empty?
 182   webhook_github_secret = get_vault_item_value(webhook_github_secret_vault_item)
 183   api_envs['WEBHOOK_GITHUB_SECRET'] = '${WEBHOOK_GITHUB_SECRET}'
 184 end
 185
 186 # ui
 187 #ui_envs_org = config_srvs['ui']['environment']
 188 ui_envs = {}
 189 ui_vols = config_srvs['ui']['volumes'].to_a
 190
 191 ui_port = '9000'  # default
 192 ui_in_port = '80'
 193 ports = config_srvs['ui']['ports']
 194 if ports.empty?
 195   override_config_srvs['ui']['ports'] = ["#{ui_port}:#{ui_in_port}"]
 196 else
 197   ports.each {|port|
 198     elms = port.split(':')
 199     ui_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == ui_in_port
 200   }
 201 end
 202
 203 # store
 204 store_envs_org = config_srvs['store']['environment']
 205 store_envs = {}
 206 store_vols = config_srvs['store']['volumes'].to_a
 207
 208 store_port = '9002'  # default
 209 store_in_port = store_envs_org['PORT']
 210 ports = config_srvs['store']['ports']
 211 if ports.empty?
 212   override_config_srvs['store']['ports'] = ["#{store_port}:#{store_in_port}"]
 213 else
 214   ports.each {|port|
 215     elms = port.split(':')
 216     store_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == store_in_port
 217   }
 218 end
 219
 220 override_store_config['auth']['jwtPublicKey'] = jwt_public_key
 221 # Note: prevent Chef from logging JWT key attribute value. (=> template variables)
 222 # However Docker env file format does not support multi-line value and backslash escaped string yet.
 223 #store_envs['SECRET_JWT_PUBLIC_KEY'] = '${SECRET_JWT_PUBLIC_KEY}'  # Useless
 224 #store_envs['SECRET_JWT_PUBLIC_KEY']  = jwt_public_key  # NG
 225
 226 api_uri = api_envs_org['URI']
 227 store_uri = store_envs_org['URI']
 228 ui_uri = api_uri.gsub(/:\d+/, ":#{ui_port}")  # based on the API URI.
 229
 230 if node['screwdriver']['with_ssl_cert_cookbook']
 231   cn = node['screwdriver']['ssl_cert']['common_name']
 232   append_server_ssl_cn(cn)
 233   include_recipe 'ssl_cert::server_key_pairs'
 234
 235   server_cert = server_cert_content(cn)
 236   server_key = server_key_content(cn)
 237
 238   api_uri = api_uri.gsub('http://', 'https://')
 239   store_uri = store_uri.gsub('http://', 'https://')
 240
 241   override_api_config['httpd']['tls'] = {}  # for FalseClass by default.
 242   override_api_config['httpd']['tls']['cert'] = server_cert
 243   override_api_config['httpd']['tls']['key'] = server_key
 244   api_envs['IS_HTTPS'] = 'true'
 245
 246   override_store_config['httpd']['tls'] = {}  # for FalseClass by default.
 247   override_store_config['httpd']['tls']['cert'] = server_cert
 248   override_store_config['httpd']['tls']['key'] = server_key
 249
 250   # Note: Screwdriver UI image does not support TLS settings yet.
 251   # https://github.com/screwdriver-cd/screwdriver/issues/377
 252
 253   if node['screwdriver']['ui']['tls_setup_mode'] == 'reverseproxy'
 254     rproxy_in_port = '9000'
 255     ports = config_srvs['reverseproxy']['ports']
 256     if ports.empty?
 257       override_config_srvs['reverseproxy']['ports'] = ["#{ui_port}:#{rproxy_in_port}"]
 258     else
 259       ports.each {|port|
 260         elms = port.split(':')
 261         ui_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == rproxy_in_port
 262       }
 263     end
 264     ui_uri = api_uri.gsub(/:\d+/, ":#{ui_port}")  # based on the API URI.
 265     # do not expose UI service directly.
 266     node.rm('screwdriver', 'docker-compose', 'config', 'services', 'ui', 'ports')
 267
 268     rproxy_vols = config_srvs['reverseproxy']['volumes'].to_a
 269     rproxy_vols.push("#{etc_dir}/nginx/nginx.conf:/etc/nginx/nginx.conf:ro")
 270     # Nginx parent process owner is root.
 271     rproxy_vols.push("#{server_cert_path(cn)}:/root/server.crt:ro")
 272     rproxy_vols.push("#{server_key_path(cn)}:/root/server.key:ro")
 273     # reset vlumes array.
 274     override_config_srvs['reverseproxy']['volumes'] = rproxy_vols
 275
 276     template "#{etc_dir}/nginx/nginx.conf" do
 277       source 'opt/docker-compose/app/screwdriver/etc/nginx/nginx.conf'
 278       owner 'root'
 279       group 'root'
 280       mode '0644'
 281       action :create
 282     end
 283   else
 284     node.rm('screwdriver', 'docker-compose', 'config', 'services', 'reverseproxy')
 285     # TODO: in the future.
 286   end
 287 else
 288   node.rm('screwdriver', 'docker-compose', 'config', 'services', 'reverseproxy')
 289 end
 290
 291 # normalize URIs
 292 api_envs['URI'] = api_uri
 293 api_envs['ECOSYSTEM_STORE'] = store_uri
 294 api_envs['ECOSYSTEM_UI'] = ui_uri
 295
 296 ui_envs['ECOSYSTEM_API'] = api_uri
 297 ui_envs['ECOSYSTEM_STORE'] = store_uri
 298
 299 store_envs['URI'] = store_uri
 300 store_envs['ECOSYSTEM_UI'] = ui_uri
 301
 302 # Common
 303 if node['screwdriver']['docker-compose']['import_ca']
 304   node['screwdriver']['ssl_cert']['ca_names'].each {|ca_name|
 305     append_ca_name(ca_name)
 306     ca_cert_vol = "#{ca_cert_path(ca_name)}:/usr/share/ca-certificates/#{ca_name}.crt:ro"
 307     api_vols.push(ca_cert_vol)
 308     #ui_vols.push(ca_cert_vol)
 309   }
 310   include_recipe 'ssl_cert::ca_certs'
 311
 312   import_ca_script = '/usr/local/bin/screwdriver_import_ca'
 313   template "#{bin_dir}/screwdriver_import_ca" do
 314     source 'opt/docker-compose/app/screwdriver/bin/screwdriver_import_ca'
 315     owner 'root'
 316     group 'root'
 317     mode '0755'
 318     action :create
 319   end
 320   import_ca_script_vol = "#{bin_dir}/screwdriver_import_ca:#{import_ca_script}:ro"
 321   api_vols.push(import_ca_script_vol)
 322   #ui_vols.push(import_ca_script_vol)
 323
 324   api_command = config_srvs['api']['command']
 325   override_config_srvs['api']['command'] \
 326     = "/bin/sh -c \"#{import_ca_script} && #{api_command}\""
 327 end
 328
 329 [
 330   'api',
 331   'store',
 332 ].each {|srv|
 333   local_yaml_file = "#{config_dir}/#{srv}-local.yaml"
 334   srv_vols = nil
 335   srv_config = nil
 336   if srv == 'api'
 337     srv_vols = api_vols
 338     srv_config = override_api_config
 339   elsif srv == 'store'
 340     srv_vols = store_vols
 341     srv_config = override_store_config
 342   end
 343
 344   template local_yaml_file do
 345     source "opt/docker-compose/app/screwdriver/config/#{srv}-local.yaml"
 346     owner 'root'
 347     group 'root'
 348     mode '0600'
 349     sensitive true
 350     # prevent Chef from logging password attribute value.
 351     variables(
 352       config: srv_config
 353     )
 354   end
 355
 356   srv_vols.push("#{local_yaml_file}:/config/local.yaml:ro")
 357 }
 358
 359 # merge environment hash
 360 force_override_config_srvs['api']['environment'] = api_envs unless api_envs.empty?
 361 force_override_config_srvs['ui']['environment'] = ui_envs unless ui_envs.empty?
 362 force_override_config_srvs['store']['environment'] = store_envs unless store_envs.empty?
 363 # reset vlumes array.
 364 override_config_srvs['api']['volumes'] = api_vols unless api_vols.empty?
 365 override_config_srvs['ui']['volumes'] = ui_vols unless ui_vols.empty?
 366 override_config_srvs['store']['volumes'] = store_vols unless store_vols.empty?
 367
 368 template env_file do
 369   source 'opt/docker-compose/app/screwdriver/.env'
 370   owner 'root'
 371   group 'root'
 372   mode '0600'
 373   sensitive true
 374   # prevent Chef from logging password attribute value.
 375   variables(
 376     # secrets
 377     # JWT keys setting -> /config/local.yaml
 378     #jwt_private_key: jwt_private_key,
 379     #jwt_public_key: jwt_public_key,
 380     cookie_password: cookie_password,
 381     password: password,
 382     oauth_client_id: oauth_client_id,
 383     oauth_client_secret: oauth_client_secret,
 384     webhook_github_secret: webhook_github_secret
 385   )
 386 end
 387
 388 template config_file do
 389   source 'opt/docker-compose/app/screwdriver/docker-compose.yml'
 390   owner 'root'
 391   group 'root'
 392   mode '0644'
 393 end
 394
 395 log <<-"EOM"
 396 Note: You must execute the following command manually.
 397   See #{doc_url}
 398   * Start:
 399     $ cd #{app_dir}
 400     $ sudo docker-compose up -d
 401   * Stop
 402     $ sudo docker-compose down
 403 EOM