From 3da74aaf1c2fc7cedf109f5bba37f36d5d54a718 Mon Sep 17 00:00:00 2001 From: whitestar Date: Sun, 24 Nov 2013 13:53:10 +0900 Subject: [PATCH] add knife cloud plugins installation recipes. --- cookbooks/chef_utils/recipes/knife-ec2.rb | 23 ++++++++ cookbooks/chef_utils/recipes/knife-eucalyptus.rb | 23 ++++++++ cookbooks/chef_utils/recipes/knife-openstack.rb | 23 ++++++++ cookbooks/hbase/attributes/default.rb | 2 +- cookbooks/hbase/recipes/default.rb | 2 +- cookbooks/pig/attributes/default.rb | 2 +- cookbooks/pig/recipes/default.rb | 12 +++- .../pig/templates/default/conf/pig.properties | 65 +++++++++++++++++++++- roles/chef_utils.rb | 1 + roles/knife-cloud-plugins.rb | 28 ++++++++++ roles/test-ah2-on-localhost.rb | 6 +- 11 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 cookbooks/chef_utils/recipes/knife-ec2.rb create mode 100644 cookbooks/chef_utils/recipes/knife-eucalyptus.rb create mode 100644 cookbooks/chef_utils/recipes/knife-openstack.rb create mode 100644 roles/knife-cloud-plugins.rb diff --git a/cookbooks/chef_utils/recipes/knife-ec2.rb b/cookbooks/chef_utils/recipes/knife-ec2.rb new file mode 100644 index 0000000..ddf3ce3 --- /dev/null +++ b/cookbooks/chef_utils/recipes/knife-ec2.rb @@ -0,0 +1,23 @@ +# +# Cookbook Name:: chef_utils +# Recipe:: knife-ec2 +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +chef_gem 'knife-ec2' do + action :install +end + diff --git a/cookbooks/chef_utils/recipes/knife-eucalyptus.rb b/cookbooks/chef_utils/recipes/knife-eucalyptus.rb new file mode 100644 index 0000000..67e67e9 --- /dev/null +++ b/cookbooks/chef_utils/recipes/knife-eucalyptus.rb @@ -0,0 +1,23 @@ +# +# Cookbook Name:: chef_utils +# Recipe:: knife-eucalyptus +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +chef_gem 'knife-eucalyptus' do + action :install +end + diff --git a/cookbooks/chef_utils/recipes/knife-openstack.rb b/cookbooks/chef_utils/recipes/knife-openstack.rb new file mode 100644 index 0000000..c5200b0 --- /dev/null +++ b/cookbooks/chef_utils/recipes/knife-openstack.rb @@ -0,0 +1,23 @@ +# +# Cookbook Name:: chef_utils +# Recipe:: knife-openstack +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +chef_gem 'knife-openstack' do + action :install +end + diff --git a/cookbooks/hbase/attributes/default.rb b/cookbooks/hbase/attributes/default.rb index 7fe21e3..57a7cab 100644 --- a/cookbooks/hbase/attributes/default.rb +++ b/cookbooks/hbase/attributes/default.rb @@ -20,7 +20,7 @@ # default: for pseudo-distributed default['hbase']['install_flavor'] = 'apache' # 0.94.2 <= version < 0.95 -default['hbase']['version'] = '0.94.8' +default['hbase']['version'] = '0.94.13' default['hbase']['archive_url'] = 'http://archive.apache.org/dist/hbase' default['hbase']['member_of_hadoop'] = false default['hbase']['with_security'] = false diff --git a/cookbooks/hbase/recipes/default.rb b/cookbooks/hbase/recipes/default.rb index 854a84f..d6ba275 100644 --- a/cookbooks/hbase/recipes/default.rb +++ b/cookbooks/hbase/recipes/default.rb @@ -63,7 +63,7 @@ else Chef::Application.fatal!("Invalid HBase version: #{version}") end -if ! ('0.94.2' <= version \ +if ! (('0.94.2' <= version || '0.94.10' <= version) \ && version <= '0.95') then Chef::Application.fatal!("Non supported version: #{version}") end diff --git a/cookbooks/pig/attributes/default.rb b/cookbooks/pig/attributes/default.rb index 604d77b..1ee46e4 100644 --- a/cookbooks/pig/attributes/default.rb +++ b/cookbooks/pig/attributes/default.rb @@ -18,7 +18,7 @@ # default['pig']['install_flavor'] = 'apache' -default['pig']['version'] = '0.11.1' +default['pig']['version'] = '0.12.0' default['pig']['archive_url'] = 'http://archive.apache.org/dist/pig' ## pig-env.sh default['pig']['PIG_PREFIX'] = "#{node['grid']['app_root']}/pig" diff --git a/cookbooks/pig/recipes/default.rb b/cookbooks/pig/recipes/default.rb index b610cda..9a5e7a5 100644 --- a/cookbooks/pig/recipes/default.rb +++ b/cookbooks/pig/recipes/default.rb @@ -79,8 +79,16 @@ if ! FileTest.directory? install_root then ruby_block "checksum #{downloaded_tarball}" do block do - # e.g. md file format 'MD5 (pig-0.11.1.tar.gz) = 1495338b54be6961106a8a5248024945' - /^.+\s+=\s+(\w+)$/ =~ File.read(downloaded_tarball_md5) + # md file format + md5sum_format = nil + if middle_version >= '0.12' then + # e.g. '78df1b943a41b83a48d02114916ba09d pig-0.12.0.tar.gz' + md5sum_format = /^(\w+)\s+.+$/ + else + # e.g. 'MD5 (pig-0.11.1.tar.gz) = 1495338b54be6961106a8a5248024945' + md5sum_format = /^.+\s+=\s+(\w+)$/ + end + md5sum_format =~ File.read(downloaded_tarball_md5) checksum = $1 Chef::Log.info "#{tarball}: MD5 = #{checksum}" actual_checksum = Digest::MD5.file(downloaded_tarball).to_s diff --git a/cookbooks/pig/templates/default/conf/pig.properties b/cookbooks/pig/templates/default/conf/pig.properties index 130ae4f..1d1ed54 100644 --- a/cookbooks/pig/templates/default/conf/pig.properties +++ b/cookbooks/pig/templates/default/conf/pig.properties @@ -145,7 +145,7 @@ hcat.bin=/usr/local/hcat/bin/hcat # By default, the logic to estimate the number of reducers to use for a given job lives in: # org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator -# This logic can be relaced by implementing the following interface: +# This logic can be replaced by implementing the following interface: # org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigReducerEstimator # This class will be invoked to estimate the number of reducers to use. @@ -156,4 +156,67 @@ hcat.bin=/usr/local/hcat/bin/hcat ##################################################################### +###### Override the default Pig Stats Output Size Reader logic ###### + +# By default, the size of reducers output is computed as the total size of +# output files. But since not every storage is file-based, this logic is not +# always applicable. If that is the case, the logic can be replaced by +# implementing the following interface: +# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigStatsOutputSizeReader + +# This class will be invoked to compute the size of reducers output. +# pig.stats.output.size.reader = + +# If you need to register more than one reader, you can register them as a comma +# separated list. Every reader implements a boolean supports(POStore sto) method. +# When there are more than one reader, they are consulted in order, and the +# first one whose supports() method returns true will be used. +# +##################################################################### + #pig.load.default.statements= + +##################################################################### + +########### Override hadoop configs programatically ################# + +# By default, Pig expects hadoop configs (hadoop-site.xml and core-site.xml) +# to be present on the classpath. There are cases when these configs are +# needed to be passed programatically, such as while using the PigServer API. +# In such cases, you can override hadoop configs by setting the property +# "pig.use.overriden.hadoop.configs". +# +# When this property is set to true, Pig ignores looking for hadoop configs +# in the classpath and instead picks it up from Properties/Configuration +# object passed to it. + +# pig.use.overriden.hadoop.configs=false +# +###################################################################### + +# Check if the script needs to check multiple stores writing +# to the same location. When set to true, stops the execution +# of script right away. +pig.location.check.strict=false + +###################################################################### + +# This key is used to define the default load func. Pig will fallback +# on PigStorage as default in case this is undefined. + +# pig.default.load.func= +# For eg, pig.default.load.func=org.apache.pig.custom.MyCustomStorage + +# This key is used to define the default store func. Pig will fallback +# on PigStorage as default in case this is undefined. + +# pig.default.store.func= +# For eg, pig.default.store.func=org.apache.pig.custom.MyCustomStorage + +# This option is used to define whether to support recovery to handle the +# application master getting restarted. +# pig.output.committer.recovery.support=true + +# Set this option to true if you need to use the old partition filter optimizer. +# Note: Old filter optimizer PColFilterOptimizer will be deprecated in the future. +# pig.exec.useOldPartitionFilterOptimize=true diff --git a/roles/chef_utils.rb b/roles/chef_utils.rb index fb2dcaa..de07a6a 100644 --- a/roles/chef_utils.rb +++ b/roles/chef_utils.rb @@ -22,6 +22,7 @@ run_list( 'role[berkshelf]', 'role[librarian-chef]', 'role[spiceweasel]', + 'role[knife-cloud-plugins]' ) #env_run_lists() diff --git a/roles/knife-cloud-plugins.rb b/roles/knife-cloud-plugins.rb new file mode 100644 index 0000000..b0a388b --- /dev/null +++ b/roles/knife-cloud-plugins.rb @@ -0,0 +1,28 @@ +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name 'knife-cloud-plugins' +description 'Knife plugins for cloud service operarions.' + +run_list( + 'recipe[chef_utils::knife-ec2]', + 'recipe[chef_utils::knife-eucalyptus]', + 'recipe[chef_utils::knife-openstack]', +) + +#env_run_lists() +#default_attributes() +#override_attributes() diff --git a/roles/test-ah2-on-localhost.rb b/roles/test-ah2-on-localhost.rb index ab52f31..ad68815 100644 --- a/roles/test-ah2-on-localhost.rb +++ b/roles/test-ah2-on-localhost.rb @@ -30,11 +30,11 @@ default_attributes( => 'org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler', }, 'hbase' => { - 'version' => '0.94.7' + 'version' => '0.94.13' }, 'pig' => { - 'version' => '0.11.1' - #'version' => '0.11.0' + 'version' => '0.12.0' + #'version' => '0.11.1' } ) -- 2.11.0