From 17ab58a26b413089e057014d1c743d20d410b2e0 Mon Sep 17 00:00:00 2001 From: whitestar Date: Sat, 3 Feb 2018 16:46:53 +0900 Subject: [PATCH] adds MySQL support. --- cookbooks/screwdriver/CHANGELOG.md | 4 + cookbooks/screwdriver/README.md | 147 +++++++++++++++++++-- cookbooks/screwdriver/attributes/default.rb | 67 +++++++++- cookbooks/screwdriver/recipes/docker-compose.rb | 71 +++++++++- .../opt/docker-compose/app/screwdriver/.env | 9 ++ cookbooks/screwdriver/version | 2 +- 6 files changed, 285 insertions(+), 15 deletions(-) diff --git a/cookbooks/screwdriver/CHANGELOG.md b/cookbooks/screwdriver/CHANGELOG.md index a8ff59f..fadfc75 100644 --- a/cookbooks/screwdriver/CHANGELOG.md +++ b/cookbooks/screwdriver/CHANGELOG.md @@ -1,5 +1,9 @@ # screwdriver CHANGELOG +0.4.0 +----- +- adds MySQL support. + 0.3.1 ----- - revises documents. diff --git a/cookbooks/screwdriver/README.md b/cookbooks/screwdriver/README.md index e3cffb6..4634bc0 100644 --- a/cookbooks/screwdriver/README.md +++ b/cookbooks/screwdriver/README.md @@ -19,8 +19,12 @@ This cookbook sets up a Screwdriver CI/CD service by Docker Compose. - [JWT private and public keys management by Chef Vault](#jwt-private-and-public-keys-management-by-chef-vault) - [Cookie password management by Chef Vault](#cookie-password-management-by-chef-vault) - [Secrets encryption password management by Chef Vault](#secrets-encryption-password-management-by-chef-vault) - - [OAuth client ID and secret management by Chef Vault](#oauth-client-id-and-secret-management-by-chef-vault) - - [GitHub webhook secret management by Chef Vault](#github-webhook-secret-management-by-chef-vault) + - [Database username management (for MySQL, PostgreSQL,...) by Chef Vault](#database-username-management-for-mysql-postgresql-by-chef-vault) + - [Database password management (for MySQL, PostgreSQL,...) by Chef Vault](#database-password-management-for-mysql-postgresql-by-chef-vault) + - [Database root password management (for MySQL, PostgreSQL,...) by Chef Vault](#database-root-password-management-for-mysql-postgresql-by-chef-vault) + - [OAuth client ID, secret and GitHub webhook secret management by Chef Vault](#oauth-client-id-secret-and-github-webhook-secret-management-by-chef-vault) + - [Note](#note) + - [Database Initialization](#database-initialization) - [License and Authors](#license-and-authors) ## Requirements @@ -48,6 +52,9 @@ This cookbook sets up a Screwdriver CI/CD service by Docker Compose. |`['screwdriver']['jwt_public_key_vault_item']`|Hash|Optional, Sets a JWT public key from Chef Vault. See `attributes/default.rb`|`{}`| |`['screwdriver']['cookie_password_vault_item']`|Hash|Optional, Sets a session cookie password from Chef Vault. See `attributes/default.rb`|`{}`| |`['screwdriver']['password_vault_item']`|Hash|Optional, Sets a password for secrets encryption from Chef Vault. See `attributes/default.rb`|`{}`| +|`['screwdriver']['db_username_vault_item']`|Hash|Optional, Sets a database username from Chef Vault. See `attributes/default.rb`|`{}`| +|`['screwdriver']['db_password_vault_item']`|Hash|Optional, Sets a database password from Chef Vault. See `attributes/default.rb`|`{}`| +|`['screwdriver']['db_root_password_vault_item']`|Hash|Optional, Sets a database password for the root user from Chef Vault. See `attributes/default.rb`|`{}`| |`['screwdriver']['ui']['tls_setup_mode']`|String|`'reverseproxy'` only. Note: [_Add TLS support to UI docker container #377_](https://github.com/screwdriver-cd/screwdriver/issues/377)|`'reverseproxy'`| |`['screwdriver']['api']['config']`|Hash|This hash object is expanded to a `/config/local.yaml` file in the API Docker container.|See `attributes/default.rb`| |`['screwdriver']['api']['scms_vault_items']`|Hash|This hash contains Chef Vault item definitions of SCM's secrets.|See `attributes/default.rb`| @@ -315,18 +322,18 @@ override_attributes( ```text $ ruby -rjson -e 'puts JSON.generate({"private" => File.read("screwdriver.io.example.com.prod.key")})' \ -> > ~/tmp/screwdriver.io.example.com.prod.key.json +> > ~/sec/tmp/screwdriver.io.example.com.prod.key.json $ ruby -rjson -e 'puts JSON.generate({"public" => File.read("screwdriver.io.example.com.prod.crt")})' \ -> > ~/tmp/screwdriver.io.example.com.prod.crt.json +> > ~/sec/tmp/screwdriver.io.example.com.prod.crt.json $ cd $CHEF_REPO_PATH $ knife vault create ssl_server_keys screwdriver.io.example.com.prod \ -> --json ~/tmp/screwdriver.io.example.com.prod.key.json +> --json ~/sec/tmp/screwdriver.io.example.com.prod.key.json $ knife vault create ssl_server_certs screwdriver.io.example.com.prod \ -> --json ~/tmp/screwdriver.io.example.com.prod.crt.json +> --json ~/sec/tmp/screwdriver.io.example.com.prod.crt.json ``` - grant reference permission to the screwdriver host @@ -411,7 +418,7 @@ override_attributes( ```text # A password used for encrypting session data. Needs to be minimum 32 characters -$ cat ~/tmp/screwdriver_cookie_password.json +$ cat ~/sec/tmp/screwdriver_cookie_password.json {"password":"********************************"} $ cd $CHEF_REPO_PATH @@ -447,7 +454,7 @@ override_attributes( ```text # A password used for encrypting stored secrets. Needs to be minimum 32 characters -$ cat ~/tmp/screwdriver_password.json +$ cat ~/sec/tmp/screwdriver_password.json {"password":"********************************"} $ cd $CHEF_REPO_PATH @@ -477,6 +484,111 @@ override_attributes( ) ``` +### Database username management (for MySQL, PostgreSQL,...) by Chef Vault + +- create vault items. + +```text +$ cat ~/sec/tmp/screwdriver_db_username.json +{"username":"********************************"} + +$ cd $CHEF_REPO_PATH +$ knife vault create screwdriver db_username --json ~/sec/tmp/screwdriver_db_username.json +``` + +- grant reference permission to the screwdriver host + +```text +$ knife vault update screwdriver db_username -S 'name:screwdriver-host.example.com' +``` + +- modify attributes + +```ruby +override_attributes( + 'screwdriver' => { + # ... + 'db_username_vault_item' => { + 'vault' => 'screwdriver', + 'name' => 'db_username', + 'env_context' => false, + 'key' => 'username', + }, + # ... + }, +) +``` + +### Database password management (for MySQL, PostgreSQL,...) by Chef Vault + +- create vault items. + +```text +$ cat ~/sec/tmp/screwdriver_db_password.json +{"password":"********************************"} + +$ cd $CHEF_REPO_PATH +$ knife vault create screwdriver db_password --json ~/sec/tmp/screwdriver_db_password.json +``` + +- grant reference permission to the screwdriver host + +```text +$ knife vault update screwdriver db_password -S 'name:screwdriver-host.example.com' +``` + +- modify attributes + +```ruby +override_attributes( + 'screwdriver' => { + # ... + 'db_password_vault_item' => { + 'vault' => 'screwdriver', + 'name' => 'db_password', + 'env_context' => false, + 'key' => 'password', + }, + # ... + }, +) +``` + +### Database root password management (for MySQL, PostgreSQL,...) by Chef Vault + +- create vault items. + +```text +$ cat ~/sec/tmp/screwdriver_db_root_password.json +{"password":"********************************"} + +$ cd $CHEF_REPO_PATH +$ knife vault create screwdriver db_root_password --json ~/sec/tmp/screwdriver_db_root_password.json +``` + +- grant reference permission to the screwdriver host + +```text +$ knife vault update screwdriver db_root_password -S 'name:screwdriver-host.example.com' +``` + +- modify attributes + +```ruby +override_attributes( + 'screwdriver' => { + # ... + 'db_root_password_vault_item' => { + 'vault' => 'screwdriver', + 'name' => 'db_root_password', + 'env_context' => false, + 'key' => 'password', + }, + # ... + }, +) +``` + ### OAuth client ID, secret and GitHub webhook secret management by Chef Vault - create vault items. @@ -538,6 +650,25 @@ override_attributes( ) ``` +### Note + +#### Database Initialization + +If you use database other than sqlite, its database initialization will takes a few tens of seconds. +You should run a database container only at the beginning and then start the others. +``` +$ sudo docker-compose up -d db +... +Creating network "screwdriver_default" with the default driver +Creating screwdriver_db_1 ... done + +$ sudo docker-compose up -d +screwdriver_db_1 is up-to-date +Creating screwdriver_api_1 ... done +Creating screwdriver_ui_1 ... done +Creating screwdriver_store_1 ... done +``` + ## License and Authors - Author:: whitestar at osdn.jp diff --git a/cookbooks/screwdriver/attributes/default.rb b/cookbooks/screwdriver/attributes/default.rb index 73c8343..4428f3c 100644 --- a/cookbooks/screwdriver/attributes/default.rb +++ b/cookbooks/screwdriver/attributes/default.rb @@ -76,6 +76,42 @@ default['screwdriver']['password_vault_item'] = { #'key' => 'hash/path/to/password', # real hash path: "/#{node.chef_environment}/hash/path/to/password" =end } +default['screwdriver']['db_username_vault_item'] = { +=begin + 'vault' => 'screwdriver', + 'name' => 'db_username', + # single usernaem or nested hash username path delimited by slash + 'env_context' => false, + 'key' => 'username', # real hash path: "/username" + # or nested hash password path delimited by slash + #'env_context' => true, + #'key' => 'hash/path/to/username', # real hash path: "/#{node.chef_environment}/hash/path/to/username" +=end +} +default['screwdriver']['db_password_vault_item'] = { +=begin + 'vault' => 'screwdriver', + 'name' => 'db_password', + # single password or nested hash password path delimited by slash + 'env_context' => false, + 'key' => 'password', # real hash path: "/password" + # or nested hash password path delimited by slash + #'env_context' => true, + #'key' => 'hash/path/to/password', # real hash path: "/#{node.chef_environment}/hash/path/to/password" +=end +} +default['screwdriver']['db_root_password_vault_item'] = { +=begin + 'vault' => 'screwdriver', + 'name' => 'db_root_password', + # single password or nested hash password path delimited by slash + 'env_context' => false, + 'key' => 'password', # real hash path: "/password" + # or nested hash password path delimited by slash + #'env_context' => true, + #'key' => 'hash/path/to/password', # real hash path: "/#{node.chef_environment}/hash/path/to/password" +=end +} force_override['screwdriver']['ui']['tls_setup_mode'] = 'reverseproxy' # These hash objects are expanded to a `/config/local.yaml` file in each Docker container. @@ -260,7 +296,7 @@ version_2_config = { 'volumes' => [ '/var/run/docker.sock:/var/run/docker.sock:rw', # This volume will be set by the screwdriver::docker-compose recipe automatically. - #"#{node['screwdriver']['docker-compose']['data_dir']}:/sd-data:rw", + #"#{node['screwdriver']['docker-compose']['data_dir']}:/sd-data:rw", # for sqlite ], 'environment' => { # See: @@ -279,8 +315,16 @@ version_2_config = { 'SECRET_WHITELIST' => '[]', 'SECRET_ADMINS' => '[]', 'DATASTORE_PLUGIN' => 'sequelize', + 'DATASTORE_SEQUELIZE_DATABASE' => 'screwdriver', 'DATASTORE_SEQUELIZE_DIALECT' => 'sqlite', - 'DATASTORE_SEQUELIZE_STORAGE' => '/sd-data/storage.db', + # This variable will be set by the screwdriver::docker-compose recipe automatically. + #'DATASTORE_SEQUELIZE_STORAGE' => '/sd-data/storage.db', + # for MySQL + #'DATASTORE_SEQUELIZE_DIALECT' => 'mysql', + # These variables will be set by the screwdriver::docker-compose recipe automatically. + #'DATASTORE_SEQUELIZE_USERNAME' => '${DB_USERNAME}', + #'DATASTORE_SEQUELIZE_PASSWORD' => '${DB_PASSWORD}', + #'DATASTORE_SEQUELIZE_HOST' => 'db', # This variable will be set by the screwdriver::docker-compose recipe automatically. #'IS_HTTPS' => 'false', #'NODE_TLS_REJECT_UNAUTHORIZED' => '0', # workaround for self-signed cetificates @@ -357,4 +401,23 @@ EOS }, } +config_srvs = node['screwdriver']['docker-compose']['config']['services'] +case config_srvs['api']['environment']['DATASTORE_SEQUELIZE_DIALECT'] +when 'mysql' + version_2_config['services']['db'] = { + 'image' => 'mysql:5', + 'volumes' => [ + # This variable will be set by the screwdriver::docker-compose recipe automatically. + #"#{node['screwdriver']['docker-compose']['data_dir']}/mysql:/var/lib/mysql:rw", + ], + 'environment' => { + # These variables will be set by the screwdriver::docker-compose recipe automatically. + #'MYSQL_ROOT_PASSWORD' => '${DB_ROOT_PASSWORD}', + #'MYSQL_USER' => '${DB_USERNAME}', + #'MYSQL_PASSWORD' => '${DB_PASSWORD}', + #'MYSQL_DATABASE' => 'screwdriver', + }, + } +end + default['screwdriver']['docker-compose']['config'] = version_2_config diff --git a/cookbooks/screwdriver/recipes/docker-compose.rb b/cookbooks/screwdriver/recipes/docker-compose.rb index 22df17a..dcf1195 100644 --- a/cookbooks/screwdriver/recipes/docker-compose.rb +++ b/cookbooks/screwdriver/recipes/docker-compose.rb @@ -111,7 +111,6 @@ else api_port = (elms.size == 2 ? elms[0] : elms[1]) if elms.last == api_in_port } end -api_vols.push("#{data_dir}:/sd-data:rw") override_api_config['executor'] = default_executor if override_api_config['executor'].empty? @@ -197,7 +196,7 @@ node['screwdriver']['api']['scms_vault_items'].each {|scm, props| } } =begin -# DEPRECATED!! +# **DEPRECATED!!** oauth_client_id = nil oauth_client_id_vault_item = node['screwdriver']['docker-compose']['oauth_client_id_vault_item'] unless oauth_client_id_vault_item.empty? @@ -220,6 +219,60 @@ unless webhook_github_secret_vault_item.empty? end =end +db_username = nil +db_username_vault_item = node['screwdriver']['db_username_vault_item'] +unless db_username_vault_item.empty? + db_username = get_vault_item_value(db_username_vault_item) + api_envs['DATASTORE_SEQUELIZE_USERNAME'] = '${DB_USERNAME}' +end + +db_password = nil +db_password_vault_item = node['screwdriver']['db_password_vault_item'] +unless db_password_vault_item.empty? + db_password = get_vault_item_value(db_password_vault_item) + api_envs['DATASTORE_SEQUELIZE_PASSWORD'] = '${DB_PASSWORD}' +end + +db_root_password = nil +db_root_password_vault_item = node['screwdriver']['db_root_password_vault_item'] +unless db_root_password_vault_item.empty? + db_root_password = get_vault_item_value(db_root_password_vault_item) +end + +db_dialect = api_envs_org['DATASTORE_SEQUELIZE_DIALECT'] +case db_dialect +when 'sqlite' + api_vols.push("#{data_dir}:/sd-data:rw") + api_envs['DATASTORE_SEQUELIZE_STORAGE'] = '/sd-data/storage.db' +when 'mysql' + override_config_srvs['api']['links'] = ['db'] + api_envs['DATASTORE_SEQUELIZE_HOST'] = 'db' +end + +# db +if db_dialect != 'sqlite' + #db_envs_org = config_srvs['db']['environment'] + db_envs = {} + db_vols = config_srvs['db']['volumes'].to_a + + case db_dialect + when 'mysql' + mysql_data_dir = "#{data_dir}/mysql" + resources(directory: mysql_data_dir) rescue directory mysql_data_dir do + owner 'root' + group 'root' + mode '0755' + recursive true + end + + db_envs['MYSQL_DATABASE'] = api_envs_org['DATASTORE_SEQUELIZE_DATABASE'] + db_envs['MYSQL_USER'] = '${DB_USERNAME}' unless db_username.nil? + db_envs['MYSQL_PASSWORD'] = '${DB_PASSWORD}' unless db_password.nil? + db_envs['MYSQL_ROOT_PASSWORD'] = '${DB_ROOT_PASSWORD}' unless db_root_password.nil? + db_vols.push("#{mysql_data_dir}:/var/lib/mysql:rw") + end +end + # ui #ui_envs_org = config_srvs['ui']['environment'] ui_envs = {} @@ -397,10 +450,16 @@ end force_override_config_srvs['api']['environment'] = api_envs unless api_envs.empty? force_override_config_srvs['ui']['environment'] = ui_envs unless ui_envs.empty? force_override_config_srvs['store']['environment'] = store_envs unless store_envs.empty? +if db_dialect != 'sqlite' + force_override_config_srvs['db']['environment'] = db_envs unless db_envs.empty? +end # reset vlumes array. override_config_srvs['api']['volumes'] = api_vols unless api_vols.empty? override_config_srvs['ui']['volumes'] = ui_vols unless ui_vols.empty? override_config_srvs['store']['volumes'] = store_vols unless store_vols.empty? +if db_dialect != 'sqlite' + override_config_srvs['db']['volumes'] = db_vols unless db_vols.empty? +end template env_file do source 'opt/docker-compose/app/screwdriver/.env' @@ -411,11 +470,15 @@ template env_file do # prevent Chef from logging password attribute value. variables( # secrets + cookie_password: cookie_password, + password: password, + db_username: db_username, + db_password: db_password, + db_root_password: db_root_password, + # **DEPRECATED!!** # JWT keys setting -> /config/local.yaml #jwt_private_key: jwt_private_key, #jwt_public_key: jwt_public_key, - cookie_password: cookie_password, - password: password, # SCM secrets setting -> /config/local.yaml #oauth_client_id: oauth_client_id, #oauth_client_secret: oauth_client_secret, diff --git a/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env b/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env index dbd3649..c4120b0 100644 --- a/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env +++ b/cookbooks/screwdriver/templates/default/opt/docker-compose/app/screwdriver/.env @@ -24,3 +24,12 @@ SECRET_OAUTH_CLIENT_SECRET=<%= @oauth_client_secret %> <% unless @webhook_github_secret.nil? %> WEBHOOK_GITHUB_SECRET=<%= @webhook_github_secret %> <% end %> +<% unless @db_username.nil? %> +DB_USERNAME=<%= @db_username %> +<% end %> +<% unless @db_password.nil? %> +DB_PASSWORD=<%= @db_password %> +<% end %> +<% unless @db_root_password.nil? %> +DB_ROOT_PASSWORD=<%= @db_root_password %> +<% end %> diff --git a/cookbooks/screwdriver/version b/cookbooks/screwdriver/version index 9e11b32..1d0ba9e 100644 --- a/cookbooks/screwdriver/version +++ b/cookbooks/screwdriver/version @@ -1 +1 @@ -0.3.1 +0.4.0 -- 2.11.0