From: whitestar <whitestar@users.osdn.me>
Date: Wed, 7 Feb 2018 11:33:13 +0000 (+0900)
Subject: adds Minio support.
X-Git-Tag: screwdriver-0.6.0
X-Git-Url: http://git.osdn.net/view?a=commitdiff_plain;h=refs%2Ftags%2Fscrewdriver-0.6.0;p=metasearch%2Fgrid-chef-repo.git

adds Minio support.
---

diff --git a/cookbooks/screwdriver/CHANGELOG.md b/cookbooks/screwdriver/CHANGELOG.md
index 6a53b41..6108ded 100644
--- a/cookbooks/screwdriver/CHANGELOG.md
+++ b/cookbooks/screwdriver/CHANGELOG.md
@@ -1,5 +1,9 @@
 # screwdriver CHANGELOG
 
+0.6.0
+-----
+- adds Minio support.
+
 0.5.0
 -----
 - adds PostgreSQL support. 
diff --git a/cookbooks/screwdriver/README.md b/cookbooks/screwdriver/README.md
index f2d5833..2550a90 100644
--- a/cookbooks/screwdriver/README.md
+++ b/cookbooks/screwdriver/README.md
@@ -5,6 +5,7 @@ This cookbook sets up a Screwdriver CI/CD service by Docker Compose.
 
 ## Contents
 
+- [Contents](#contents)
 - [Requirements](#requirements)
     - [platforms](#platforms)
     - [packages](#packages)
@@ -22,6 +23,7 @@ This cookbook sets up a Screwdriver CI/CD service by Docker Compose.
     - [Database username management (for MySQL, PostgreSQL,...) by Chef Vault](#database-username-management-for-mysql-postgresql-by-chef-vault)
     - [Database password management (for MySQL, PostgreSQL,...) by Chef Vault](#database-password-management-for-mysql-postgresql-by-chef-vault)
     - [Database root password management (for MySQL, PostgreSQL,...) by Chef Vault](#database-root-password-management-for-mysql-postgresql-by-chef-vault)
+    - [S3 (compatible) server access key management by Chef Vault](#s3-compatible-server-access-key-management-by-chef-vault)
     - [OAuth client ID, secret and GitHub webhook secret management by Chef Vault](#oauth-client-id-secret-and-github-webhook-secret-management-by-chef-vault)
     - [Note](#note)
         - [Database Initialization](#database-initialization)
@@ -55,9 +57,12 @@ This cookbook sets up a Screwdriver CI/CD service by Docker Compose.
 |`['screwdriver']['db_username_vault_item']`|Hash|Optional, Sets a database username from Chef Vault. See `attributes/default.rb`|`{}`|
 |`['screwdriver']['db_password_vault_item']`|Hash|Optional, Sets a database password from Chef Vault. See `attributes/default.rb`|`{}`|
 |`['screwdriver']['db_root_password_vault_item']`|Hash|Optional, Sets a database password for the root user from Chef Vault. See `attributes/default.rb`|`{}`|
+|`['screwdriver']['s3_access_key_id_vault_item']`|Hash|Optional, Sets a S3 access key id from Chef Vault. See `attributes/default.rb`|`{}`|
+|`['screwdriver']['s3_access_key_secret_vault_item']`|Hash|Optional, Sets a S3 access key secret from Chef Vault. See `attributes/default.rb`|`{}`|
 |`['screwdriver']['ui']['tls_setup_mode']`|String|`'reverseproxy'` only. Note: [_Add TLS support to UI docker container #377_](https://github.com/screwdriver-cd/screwdriver/issues/377)|`'reverseproxy'`|
 |`['screwdriver']['api']['config']`|Hash|This hash object is expanded to a `/config/local.yaml` file in the API Docker container.|See `attributes/default.rb`|
 |`['screwdriver']['api']['scms_vault_items']`|Hash|This hash contains Chef Vault item definitions of SCM's secrets.|See `attributes/default.rb`|
+|`['screwdriver']['store']['backend']`|String|`nil` (in memory) or `'minio'`.|`nil`|
 |`['screwdriver']['store']['config']`|Hash|This hash object is expanded to a `/config/local.yaml` file in the Store Docker container.|See `attributes/default.rb`|
 |`['screwdriver']['docker-compose']['import_ca']`|Boolean|whether import internal CA certificates or not.|`false`|
 |`['screwdriver']['docker-compose']['app_dir']`|String|Path string.|`"#{node['docker-grid']['compose']['app_dir']}/screwdriver"`|
@@ -589,6 +594,50 @@ override_attributes(
 )
 ```
 
+### S3 (compatible) server access key management by Chef Vault
+
+- create vault items.
+
+```text
+$ cat ~/sec/tmp/screwdriver_s3_access_key.json
+{
+  "kid":"********************",
+  "secret":"****************************************"
+}
+
+$ cd $CHEF_REPO_PATH
+$ knife vault create screwdriver s3_access_key --json ~/sec/tmp/screwdriver_s3_access_key.json
+```
+
+- grant reference permission to the screwdriver host
+
+```text
+$ knife vault update screwdriver s3_access_key -S 'name:screwdriver-host.example.com'
+```
+
+- modify attributes
+
+```ruby
+override_attributes(
+  'screwdriver' => {
+    # ...
+    's3_access_key_id_vault_item' => {
+      'vault' => 'screwdriver',
+      'name' => 's3_access_key',
+      'env_context' => false,
+      'key' => 'kid',
+    },
+    's3_access_key_secret_vault_item' => {
+      'vault' => 'screwdriver',
+      'name' => 's3_access_key',
+      'env_context' => false,
+      'key' => 'secret',
+    },
+    # ...
+  },
+)
+```
+
 ### OAuth client ID, secret and GitHub webhook secret management by Chef Vault
 
 - create vault items.
diff --git a/cookbooks/screwdriver/attributes/default.rb b/cookbooks/screwdriver/attributes/default.rb
index beabd24..6a65439 100644
--- a/cookbooks/screwdriver/attributes/default.rb
+++ b/cookbooks/screwdriver/attributes/default.rb
@@ -30,10 +30,10 @@ default['screwdriver']['jwt_private_key_vault_item'] = {
 =begin
   'vault' => 'screwdriver',
   'name' => 'jwt_private_key',
-  # single password or nested hash password path delimited by slash
+  # single secret or nested hash secret path delimited by slash
   'env_context' => false,
-  'key' => 'private',  # real hash path: "/password"
-  # or nested hash password path delimited by slash
+  'key' => 'private',  # real hash path: "/private"
+  # or nested hash secret path delimited by slash
   #'env_context' => true,
   #'key' => 'hash/path/to/private',  # real hash path: "/#{node.chef_environment}/hash/path/to/private"
 =end
@@ -42,10 +42,10 @@ default['screwdriver']['jwt_public_key_vault_item'] = {
 =begin
   'vault' => 'screwdriver',
   'name' => 'jwt_public_key',
-  # single password or nested hash password path delimited by slash
+  # single secret or nested hash secret path delimited by slash
   'env_context' => false,
-  'key' => 'public',  # real hash path: "/password"
-  # or nested hash password path delimited by slash
+  'key' => 'public',  # real hash path: "/public"
+  # or nested hash secret path delimited by slash
   #'env_context' => true,
   #'key' => 'hash/path/to/public',  # real hash path: "/#{node.chef_environment}/hash/path/to/public"
 =end
@@ -83,7 +83,7 @@ default['screwdriver']['db_username_vault_item'] = {
   # single usernaem or nested hash username path delimited by slash
   'env_context' => false,
   'key' => 'username',  # real hash path: "/username"
-  # or nested hash password path delimited by slash
+  # or nested hash username path delimited by slash
   #'env_context' => true,
   #'key' => 'hash/path/to/username',  # real hash path: "/#{node.chef_environment}/hash/path/to/username"
 =end
@@ -112,6 +112,30 @@ default['screwdriver']['db_root_password_vault_item'] = {
   #'key' => 'hash/path/to/password',  # real hash path: "/#{node.chef_environment}/hash/path/to/password"
 =end
 }
+default['screwdriver']['s3_access_key_id_vault_item'] = {
+=begin
+  'vault' => 'screwdriver',
+  'name' => 's3_access_key',
+  # single key id or nested hash key id path delimited by slash
+  'env_context' => false,
+  'key' => 'kid',  # real hash path: "/kid"
+  # or nested hash key id path delimited by slash
+  #'env_context' => true,
+  #'key' => 'hash/path/to/kid',  # real hash path: "/#{node.chef_environment}/hash/path/to/kid"
+=end
+}
+default['screwdriver']['s3_access_key_secret_vault_item'] = {
+=begin
+  'vault' => 'screwdriver',
+  'name' => 's3_access_key',
+  # single secret or nested hash secret path delimited by slash
+  'env_context' => false,
+  'key' => 'secret',  # real hash path: "/secret"
+  # or nested hash secret path delimited by slash
+  #'env_context' => true,
+  #'key' => 'hash/path/to/secret',  # real hash path: "/#{node.chef_environment}/hash/path/to/secret"
+=end
+}
 
 force_override['screwdriver']['ui']['tls_setup_mode'] = 'reverseproxy'
 # These hash objects are expanded to a `/config/local.yaml` file in each Docker container.
@@ -201,11 +225,26 @@ default['screwdriver']['api']['scms_vault_items'] = {
 =end
 }
 
+default['screwdriver']['store']['backend'] = nil  # or 'minio'
 default['screwdriver']['store']['config'] = {
   'auth' => {},
   'httpd' => {
     'tls' => false,
   },
+=begin
+  # for Minio
+  'strategy' => {
+    'plugin' => 's3',
+    's3' => {
+      'accessKeyId' => '',
+      'secretAccessKey' => '****************************************',
+      'region' => 'us-east-1',
+      'bucket' => 'screwdriver',
+      'endpoint' => 'http://s3:9000/screwdriver',
+      'signatureVersion' => 'v4',
+    },
+  },
+=end
 }
 
 # Useless?!
@@ -391,12 +430,34 @@ EOS
         'PORT' => '80',
         'URI' => "http://#{cn}:9002",
         #'URI' => "http://#{node['ipaddress']}:9002",  # unrecommended
-        #'STRATEGY' => 'memory',
-        # This variable will be set by the screwdriver::docker-compose recipe automatically.
+        # These variables will be set by the screwdriver::docker-compose recipe automatically.
         #'ECOSYSTEM_UI' => "http://#{cn}:9000",  # Better
         #'ECOSYSTEM_UI' => "http://#{node['ipaddress']}:9000",
         #'ECOSYSTEM_UI' => 'http://ui',          # NG for an access from a client.
+        #'STRATEGY' => 'memory',  # default
+        # * AWS S3
+        #'STRATEGY' => 's3',
+        # If node['screwdriver']['s3_access_key_{id,secret}_vault_item'] is set,
+        # these 2 variables will be set by the screwdriver::docker-compose recipe automatically.
+        #'S3_ACCESS_KEY_ID' => '${S3_ACCESS_KEY_ID}',
+        #'S3_ACCESS_KEY_SECRET' => '${S3_ACCESS_KEY_SECRET}',
+        #'S3_REGION' => 'us-east-1',
+        #'S3_BUCKET' => 'screwdriver',
+        # * Minio
+        # If node['screwdriver']['store']['backend'] is 'minio',
+        # these variables will be set by the screwdriver::docker-compose recipe automatically.
+        #'STRATEGY' => 's3',
+        #'S3_ACCESS_KEY_ID' => '${S3_ACCESS_KEY_ID}',
+        #'S3_ACCESS_KEY_SECRET' => '${S3_ACCESS_KEY_SECRET}',
+        #'S3_REGION' => 'us-east-1',
+        #'S3_BUCKET' => 'screwdriver',
+        #'S3_ENDPOINT' => 'http://s3:9000/screwdriver',  # tricky!! setting for the S3 virtual hosting style.
+        #'S3_SIG_VER' => 'v4',
       },
+      # for S3 compatible server
+      #'links' => [
+      #  'screwdriver.s3',
+      #],
     },
   },
 }
@@ -435,4 +496,25 @@ when 'postgres'
   }
 end
 
+# S3 compatible server
+case node['screwdriver']['store']['backend']
+when 'minio'
+  version_2_config['services']['screwdriver.s3'] = {
+    'image' => 'minio/minio',
+    'ports' => [
+      #'9010:9000',  # default
+    ],
+    'command' => 'server /export',
+    'volumes' => [
+      # This variable will be set by the screwdriver::docker-compose recipe automatically.
+      #"#{node['screwdriver']['docker-compose']['data_dir']}//minio:/export:rw",
+    ],
+    'environment' => {
+      # These variables will be set by the screwdriver::docker-compose recipe automatically.
+      #'MINIO_ACCESS_KEY' => '${S3_ACCESS_KEY_ID}',
+      #'MINIO_SECRET_KEY' => '${S3_ACCESS_KEY_SECRET}',
+    },
+  }
+end
+
 default['screwdriver']['docker-compose']['config'] = version_2_config
diff --git a/cookbooks/screwdriver/recipes/docker-compose.rb b/cookbooks/screwdriver/recipes/docker-compose.rb
index f926763..5a0b0e5 100644
--- a/cookbooks/screwdriver/recipes/docker-compose.rb
+++ b/cookbooks/screwdriver/recipes/docker-compose.rb
@@ -259,17 +259,17 @@ if db_dialect != 'sqlite'
   when 'mysql'
     mysql_data_dir = "#{data_dir}/mysql"
     resources(directory: mysql_data_dir) rescue directory mysql_data_dir do
-      owner 'root'
-      group 'root'
+      owner 999
+      group 'docker'
       mode '0755'
       recursive true
     end
 
+    db_vols.push("#{mysql_data_dir}:/var/lib/mysql:rw")
     db_envs['MYSQL_DATABASE'] = api_envs_org['DATASTORE_SEQUELIZE_DATABASE']
     db_envs['MYSQL_USER'] = '${DB_USERNAME}' unless db_username.nil?
     db_envs['MYSQL_PASSWORD'] = '${DB_PASSWORD}' unless db_password.nil?
     db_envs['MYSQL_ROOT_PASSWORD'] = '${DB_ROOT_PASSWORD}' unless db_root_password.nil?
-    db_vols.push("#{mysql_data_dir}:/var/lib/mysql:rw")
   when 'postgres'
     pg_data_dir = "#{data_dir}/postgres"
     resources(directory: pg_data_dir) rescue directory pg_data_dir do
@@ -279,11 +279,11 @@ if db_dialect != 'sqlite'
       recursive true
     end
 
+    db_vols.push("#{pg_data_dir}:/database:rw")
     db_envs['POSTGRES_DB'] = api_envs_org['DATASTORE_SEQUELIZE_DATABASE']
     db_envs['POSTGRES_USER'] = '${DB_USERNAME}' unless db_username.nil?
     db_envs['POSTGRES_PASSWORD'] = '${DB_PASSWORD}' unless db_password.nil?
     db_envs['PGDATA'] = '/database'
-    db_vols.push("#{pg_data_dir}:/database:rw")
   end
 end
 
@@ -305,6 +305,7 @@ else
 end
 
 # store
+store_backend = node['screwdriver']['store']['backend']
 store_envs_org = config_srvs['store']['environment']
 store_envs = {}
 store_vols = config_srvs['store']['volumes'].to_a
@@ -321,6 +322,63 @@ else
   }
 end
 
+s3_access_key_id = nil
+s3_access_key_id_vault_item = node['screwdriver']['s3_access_key_id_vault_item']
+unless s3_access_key_id_vault_item.empty?
+  s3_access_key_id = get_vault_item_value(s3_access_key_id_vault_item)
+  store_envs['S3_ACCESS_KEY_ID'] = '${S3_ACCESS_KEY_ID}'
+end
+
+s3_access_key_secret = nil
+s3_access_key_secret_vault_item = node['screwdriver']['s3_access_key_secret_vault_item']
+unless s3_access_key_secret_vault_item.empty?
+  s3_access_key_secret = get_vault_item_value(s3_access_key_secret_vault_item)
+  store_envs['S3_ACCESS_KEY_SECRET'] = '${S3_ACCESS_KEY_SECRET}'
+end
+
+# S3 compatible server
+if !store_backend.nil? && !store_backend.empty?
+  override_config_srvs['store']['links'] = ['screwdriver.s3']
+  store_envs['STRATEGY'] = 's3'
+  store_envs['S3_BUCKET'] = 'screwdriver'
+
+  #s3_envs_org = config_srvs['screwdriver.s3']['environment']
+  s3_envs = {}
+  s3_vols = config_srvs['screwdriver.s3']['volumes'].to_a
+
+  s3_port = '9010'  # default
+  s3_in_port = '9000'
+  ports = config_srvs['screwdriver.s3']['ports']
+
+  case store_backend
+  when 'minio'
+    store_envs['S3_REGION'] = 'us-east-1'
+    store_envs['S3_ENDPOINT'] = "http://s3:#{s3_in_port}/screwdriver"  # for path style
+    store_envs['S3_SIG_VER'] = 'v4'
+
+    if ports.empty?
+      override_config_srvs['screwdriver.s3']['ports'] = ["#{s3_port}:#{s3_in_port}"]
+    else
+      ports.each {|port|
+        elms = port.split(':')
+        s3_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == s3_in_port
+      }
+    end
+
+    minio_data_dir = "#{data_dir}/minio"
+    resources(directory: minio_data_dir) rescue directory minio_data_dir do
+      owner 'root'
+      group 'root'
+      mode '0755'
+      recursive true
+    end
+
+    s3_vols.push("#{minio_data_dir}:/export:rw")
+    s3_envs['MINIO_ACCESS_KEY'] = '${S3_ACCESS_KEY_ID}' unless s3_access_key_id.nil?
+    s3_envs['MINIO_SECRET_KEY'] = '${S3_ACCESS_KEY_SECRET}' unless s3_access_key_secret.nil?
+  end
+end
+
 override_store_config['auth']['jwtPublicKey'] = jwt_public_key
 # Note: prevent Chef from logging JWT key attribute value. (=> template variables)
 # However Docker env file format does not support multi-line value and backslash escaped string yet.
@@ -467,6 +525,9 @@ force_override_config_srvs['store']['environment'] = store_envs unless store_env
 if db_dialect != 'sqlite'
   force_override_config_srvs['db']['environment'] = db_envs unless db_envs.empty?
 end
+if !store_backend.nil? && !store_backend.empty?
+  force_override_config_srvs['screwdriver.s3']['environment'] = s3_envs unless s3_envs.empty?
+end
 # reset vlumes array.
 override_config_srvs['api']['volumes'] = api_vols unless api_vols.empty?
 override_config_srvs['ui']['volumes'] = ui_vols unless ui_vols.empty?
@@ -474,6 +535,9 @@ override_config_srvs['store']['volumes'] = store_vols unless store_vols.empty?
 if db_dialect != 'sqlite'
   override_config_srvs['db']['volumes'] = db_vols unless db_vols.empty?
 end
+if !store_backend.nil? && !store_backend.empty?
+  override_config_srvs['screwdriver.s3']['volumes'] = s3_vols unless s3_vols.empty?
+end
 
 template env_file do
   source 'opt/docker-compose/app/screwdriver/.env'
@@ -489,6 +553,8 @@ template env_file do
     db_username: db_username,
     db_password: db_password,
     db_root_password: db_root_password,
+    s3_access_key_id: s3_access_key_id,
+    s3_access_key_secret: s3_access_key_secret,
     # **DEPRECATED!!**
     # JWT keys setting -> /config/local.yaml
     #jwt_private_key: jwt_private_key,
diff --git a/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env b/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env
index c4120b0..a9ab2d5 100644
--- a/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env
+++ b/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env
@@ -33,3 +33,9 @@ DB_PASSWORD=<%= @db_password %>
 <% unless @db_root_password.nil? %>
 DB_ROOT_PASSWORD=<%= @db_root_password %>
 <% end %>
+<% unless @s3_access_key_id.nil? %>
+S3_ACCESS_KEY_ID=<%= @s3_access_key_id %>
+<% end %>
+<% unless @s3_access_key_secret.nil? %>
+S3_ACCESS_KEY_SECRET=<%= @s3_access_key_secret %>
+<% end %>
diff --git a/cookbooks/screwdriver/version b/cookbooks/screwdriver/version
index 8f0916f..a918a2a 100644
--- a/cookbooks/screwdriver/version
+++ b/cookbooks/screwdriver/version
@@ -1 +1 @@
-0.5.0
+0.6.0