ruby_block "sha256 checksum #{downloaded_hadoop_tarball}" do
block do
+=begin
+e.g. md file format
+'hadoop-1.1.2-bin.tar.gz: MD5 = 4B 59 F4 81 A7 52 D2 A9 20 3D D7 D0 A9 50 5C
+ 18
+hadoop-1.1.2-bin.tar.gz: SHA1 = DCCC 01A0 4C42 587D 9DF1 83CA 7DC8 83F7 A6A4
+ 8D80
+...'
+=end
sha256checksum = File.read(downloaded_hadoop_tarball_mds).
gsub(/(\s)+/, '').
scan(/#{hadoop_tarball}:(.+?)=([0-9A-Z]+)/).
assoc('SHA256')[1]
Chef::Log.info "#{hadoop_tarball}: SHA256 = #{sha256checksum}"
- if ! Digest::SHA256.file(downloaded_hadoop_tarball).to_s.casecmp(sha256checksum) then
+ actual_sha256checksum = Digest::SHA256.file(downloaded_hadoop_tarball).to_s
+ Chef::Log.info "#{hadoop_tarball}: actual SHA256 = #{actual_sha256checksum}"
+ if ! sha256checksum.casecmp(actual_sha256checksum) then
Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_hadoop_tarball}, expected: #{sha256checksum}")
end
end
ruby_block "checksum #{downloaded_hbase_tarball}" do
block do
+ # e.g. md file format '8fb29417b1f6a0dd6a939719e92937e7 hbase-0.94.8.tar.gz'
/^(\w+)\s+.*$/ =~ File.read(downloaded_hbase_tarball_md5)
md5checksum = $1
Chef::Log.info "#{hbase_tarball}: MD5 = #{md5checksum}"
- if ! Digest::MD5.file(downloaded_hbase_tarball).to_s.casecmp(md5checksum) then
+ actual_md5checksum = Digest::MD5.file(downloaded_hbase_tarball).to_s
+ Chef::Log.info "#{hbase_tarball}: actual MD5 = #{actual_md5checksum}"
+ if ! md5checksum.casecmp(actual_md5checksum) then
Chef::Application.fatal!("Invalid MD5 checksum of #{downloaded_hbase_tarball}, expected: #{md5checksum}")
end
=begin
--- /dev/null
+# CHANGELOG for pig
+
+This file is used to list changes made in each version of pig.
+
+## 0.1.0:
+
+* Initial release of pig
+
+- - -
+Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown.
+
+The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown.
--- /dev/null
+pig Cookbook
+============
+TODO: Enter the cookbook description here.
+
+e.g.
+This cookbook makes your favorite breakfast sandwhich.
+
+Requirements
+------------
+TODO: List your cookbook requirements. Be sure to include any requirements this cookbook has on platforms, libraries, other cookbooks, packages, operating systems, etc.
+
+e.g.
+#### packages
+- `toaster` - pig needs toaster to brown your bagel.
+
+Attributes
+----------
+TODO: List you cookbook attributes here.
+
+e.g.
+#### pig::default
+<table>
+ <tr>
+ <th>Key</th>
+ <th>Type</th>
+ <th>Description</th>
+ <th>Default</th>
+ </tr>
+ <tr>
+ <td><tt>['pig']['bacon']</tt></td>
+ <td>Boolean</td>
+ <td>whether to include bacon</td>
+ <td><tt>true</tt></td>
+ </tr>
+</table>
+
+Usage
+-----
+#### pig::default
+TODO: Write usage instructions for each cookbook.
+
+e.g.
+Just include `pig` in your node's `run_list`:
+
+```json
+{
+ "name":"my_node",
+ "run_list": [
+ "recipe[pig]"
+ ]
+}
+```
+
+Contributing
+------------
+TODO: (optional) If this is a public cookbook, detail the process for contributing. If this is a private cookbook, remove this section.
+
+e.g.
+1. Fork the repository on Github
+2. Create a named feature branch (like `add_component_x`)
+3. Write you change
+4. Write tests for your change (if applicable)
+5. Run the tests, ensuring they all pass
+6. Submit a Pull Request using Github
+
+License and Authors
+-------------------
+Authors: TODO: List authors
--- /dev/null
+#
+# Cookbook Name:: pig
+# Attributes:: default
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+default['pig']['install_flavor'] = 'apache'
+default['pig']['version'] = '0.11.1'
+default['pig']['archive_url'] = 'http://archive.apache.org/dist/pig'
+## pig-env.sh
+default['pig']['PIG_PREFIX'] = "#{node['grid']['app_root']}/pig"
+default['pig']['HADOOP_HOME'] = "#{node['hadoop']['HADOOP_PREFIX']}"
+default['pig']['HADOOP_CONF_DIR'] = '${HADOOP_HOME}/conf'
+## extra settings
+default['pig']['extra_configs'] = {
+ # e.g. 'pig-env.xml' => {'k1' => 'v1', 'k2' => 'v2'},
+ 'pig-env.sh' => {}
+}
+#default['pig'][''] =
+
--- /dev/null
+name 'pig'
+maintainer 'YOUR_COMPANY_NAME'
+maintainer_email 'YOUR_EMAIL'
+license 'All rights reserved'
+description 'Installs/Configures pig'
+long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
+version '0.1.0'
--- /dev/null
+#
+# Cookbook Name:: pig
+# Recipe:: default
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'digest/md5'
+
+pig_major_version = ''
+pig_middle_version = ''
+
+if /^(\d+)\.(\d+)\.(\d+)$/ =~ node['pig']['version'] then
+ pig_major_version = $1
+ pig_middle_version = "#{$1}.#{$2}"
+else
+ Chef::Application.fatal!("Invalid Pig version: #{node['pig']['version']}")
+end
+
+def conf_template(conf_dir, conf_files, tpl_vars)
+ conf_files.each {|conf_file|
+ template "#{conf_dir}/#{conf_file}" do
+ source "conf/#{conf_file}"
+ #source "conf-#{pig_middle_version}/#{conf_file}"
+ owner 'root'
+ group 'root'
+ mode '0644'
+ variables(tpl_vars)
+ end
+ }
+end
+
+pig_install_root = "#{node['grid']['app_root']}/pig-#{node['pig']['version']}"
+pig_tarball = "pig-#{node['pig']['version']}.tar.gz"
+pig_tarball_md5 = "#{pig_tarball}.md5"
+#pig_tarball_mds = "#{pig_tarball}.mds"
+downloaded_pig_tarball = "#{Chef::Config[:file_cache_path]}/#{pig_tarball}"
+downloaded_pig_tarball_md5 = "#{Chef::Config[:file_cache_path]}/#{pig_tarball_md5}"
+#downloaded_pig_tarball_mds = "#{Chef::Config[:file_cache_path]}/#{pig_tarball_mds}"
+
+if ! FileTest.directory? pig_install_root then
+ remote_file downloaded_pig_tarball_md5 do
+ source "#{node['pig']['archive_url']}/pig-#{node['pig']['version']}/#{pig_tarball_md5}"
+ action :create_if_missing
+ end
+=begin
+ remote_file downloaded_pig_tarball_mds do
+ source "#{node['pig']['archive_url']}/pig-#{node['pig']['version']}/#{pig_tarball_mds}"
+ action :create_if_missing
+ end
+=end
+
+ remote_file downloaded_pig_tarball do
+ source "#{node['pig']['archive_url']}/pig-#{node['pig']['version']}/#{pig_tarball}"
+ action :create_if_missing
+ end
+
+ ruby_block "checksum #{downloaded_pig_tarball}" do
+ block do
+ # e.g. md file format 'MD5 (pig-0.11.1.tar.gz) = 1495338b54be6961106a8a5248024945'
+ /^.+\s+=\s+(\w+)$/ =~ File.read(downloaded_pig_tarball_md5)
+ md5checksum = $1
+ Chef::Log.info "#{pig_tarball}: MD5 = #{md5checksum}"
+ actual_md5checksum = Digest::MD5.file(downloaded_pig_tarball).to_s
+ Chef::Log.info "#{pig_tarball}: actual MD5 = #{actual_md5checksum}"
+ if ! md5checksum.casecmp(actual_md5checksum) then
+ Chef::Application.fatal!("Invalid MD5 checksum of #{downloaded_pig_tarball}, expected: #{md5checksum}")
+ end
+=begin
+ sha256checksum = File.read(downloaded_pig_tarball_mds).
+ gsub(/(\s)+/, '').
+ scan(/#{pig_tarball}:(.+?)=([0-9A-Z]+)/).
+ assoc('SHA256')[1]
+ Chef::Log.info "#{pig_tarball}: SHA256 = #{sha256checksum}"
+ if ! Digest::SHA256.file(downloaded_pig_tarball).to_s.casecmp(sha256checksum) then
+ Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_pig_tarball}, expected: #{sha256checksum}")
+ end
+=end
+ end
+ action :create
+ end
+
+ package 'tar' do
+ action :install
+ end
+
+ bash "install_pig: #{pig_install_root}" do
+ code <<-EOC
+ tar xvzf #{downloaded_pig_tarball} -C #{node['grid']['app_root']}
+ EOC
+ creates pig_install_root
+ end
+
+ link node['pig']['PIG_PREFIX'] do
+ to pig_install_root
+ end
+end
+
+conf_files = [
+ 'pig-env.sh',
+ 'pig.properties',
+ 'log4j.properties',
+ 'log4j.properties.template'
+]
+
+conf_dir = "#{pig_install_root}/conf"
+tpl_vars = {}
+conf_template(conf_dir, conf_files, tpl_vars)
+
--- /dev/null
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
--- /dev/null
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
--- /dev/null
+export JAVA_HOME=<%= node['java']['java_home'] %>
+
+export HADOOP_HOME_WARN_SUPPRESS=true
+export HADOOP_HOME=<%= node['pig']['HADOOP_HOME'] %>
+export HADOOP_CONF_DIR=<%= node['pig']['HADOOP_CONF_DIR'] %>
+# for Hadoop deb and rpm package.
+#export HADOOP_HOME=/usr/share/hadoop
+#export HADOOP_CONF_DIR=/etc/hadoop
+
+<%
+this_file = 'pig-env.sh'
+if defined? node['pig']['extra_configs'][this_file] \
+ && node['pig']['extra_configs'][this_file] != nil then
+ node['pig']['extra_configs'][this_file].each do |key, value|
+%>
+export <%= key %>=<%= value %>
+<%
+ end
+end
+%>
+
--- /dev/null
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig configuration file. All values can be overwritten by command line arguments.
+
+# Use the "-h properties" command to see description of the properties
+
+# log4jconf log4j configuration file
+# log4jconf=./conf/log4j.properties
+
+# a file that contains pig script
+#file=
+
+# load jarfile, colon separated
+#jar=
+
+#verbose print all log messages to screen (default to print only INFO and above to screen)
+#verbose=true
+
+#exectype local|mapreduce, mapreduce is default
+#exectype=local
+
+#the default timezone: if it is not set, the default timezone for this host is used.
+#the correct timezone format is the UTC offset: e.g., +08:00.
+#pig.datetime.default.tz=
+
+#pig.logfile=
+
+#Do not spill temp files smaller than this size (bytes)
+#pig.spill.size.threshold=5000000
+
+#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+#This should help reduce the number of files being spilled.
+#pig.spill.gc.activation.size=40000000
+
+#the following two parameters are to help estimate the reducer number
+#pig.exec.reducers.bytes.per.reducer=1000000000
+#pig.exec.reducers.max=999
+
+#Logging properties
+#verbose=false
+#brief=false
+#debug=INFO
+#aggregate.warning=true
+
+#Performance tuning properties
+#pig.cachedbag.memusage=0.2
+#pig.skewedjoin.reduce.memusagea=0.3
+#pig.exec.nocombiner=false
+#opt.multiquery=true
+#pig.tmpfilecompression=false
+
+#value can be lzo or gzip
+#pig.tmpfilecompression.codec=gzip
+#pig.noSplitCombination=true
+
+#pig.exec.mapPartAgg=false
+#pig.exec.mapPartAgg.minReduction=10
+
+#exectype=mapreduce
+#pig.additional.jars=<comma seperated list of jars>
+#udf.import.list=<comma seperated list of imports>
+#stop.on.failure=false
+
+#Use this option only when your Pig job will otherwise die because of
+#using more counters than hadoop configured limit
+#pig.disable.counter=true
+
+# Use this option to turn on UDF timers. This will cause two
+# counters to be tracked for every UDF and LoadFunc in your script:
+# approx_microsecs measures approximate time spent inside a UDF
+# approx_invocations reports the approximate number of times the UDF was invoked
+# pig.udf.profile=false
+
+#When enabled, 'describe' prints a multi-line formatted schema
+#(similar to an indended json) rather than on a single line.
+#pig.pretty.print.schema=true
+
+#pig.sql.type=hcat
+hcat.bin=/usr/local/hcat/bin/hcat
+
+############################ SchemaTuple ############################
+
+# Setting this value will turn on the SchemaTuple feature (PIG-2632)
+# This will attempt to use code generation for more efficient within
+# the pig code. This can lead to both CPU, serialization, and memory
+# benefits (currently, the potential memory benefits are the largest).
+
+# This parameter will enable the optimization in all available cases
+#pig.schematuple=true
+
+# Certain cases can be turned off by uncommenting the following. These will
+# all be off by default, but will all be turned on if pig.schematuple is set
+# to true.
+
+# This will disable SchemaTuples in the case of udfs. Currently,
+# the input to UDF's will be SchemaTuples.
+
+#pig.schematuple.udf=false
+
+# This is currently not implemented. In the future, LoadFunc's with known
+# schema's should output SchemaTuples
+
+#pig.schematuple.load=false
+
+# This will use SchemaTuples in replicated joins. The potential memory saving
+# here is significant. It will use SchemaTuples when it builds the HashMap of
+# the join key to related values.
+
+#pig.schematuple.fr_join=false
+
+# In the current implementation of merge join, all of the Tuples in the left relation
+# that share a given key will be stored in a List in memory. This will use SchemaTuples
+# instead in that List.
+
+#pig.schematuple.merge_join=false
+
+#####################################################################
+
+##### Set up optional Pig Progress Notification Listener ############
+
+# Note that only one PPNL can be set up. If you need several, write a PPNL that will chain them.
+# pig.notification.listener = <fully qualified class name of a PPNL implementation>
+
+# Optionally, you can supply a single String argument to pass to your PPNL.
+# pig.notification.listener.arg = <somevalue>
+
+#####################################################################
+
+########## Override the default Reducer Estimator logic #############
+
+# By default, the logic to estimate the number of reducers to use for a given job lives in:
+# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator
+# This logic can be relaced by implementing the following interface:
+# org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigReducerEstimator
+
+# This class will be invoked to estimate the number of reducers to use.
+# pig.exec.reducer.estimator = <fully qualified class name of a PigReducerEstimator implementation>
+
+# Optionally, you can supply a single String argument to pass to your PigReducerEstimator.
+# pig.exec.reducer.estimator.arg = <somevalue>
+
+#####################################################################
+
+#pig.load.default.statements=
ruby_block "checksum #{downloaded_zookeeper_tarball}" do
block do
+ # e.g. md file format 'f64fef86c0bf2e5e0484d19425b22dcb zookeeper-3.4.5.tar.gz'
/^(\w+)\s+.*$/ =~ File.read(downloaded_zookeeper_tarball_md5)
md5checksum = $1
Chef::Log.info "#{zookeeper_tarball}: MD5 = #{md5checksum}"
- if ! Digest::MD5.file(downloaded_zookeeper_tarball).to_s.casecmp(md5checksum) then
+ actual_md5checksum = Digest::MD5.file(downloaded_zookeeper_tarball).to_s
+ Chef::Log.info "#{zookeeper_tarball}: actual MD5 = #{actual_md5checksum}"
+ if ! md5checksum.casecmp(actual_md5checksum) then
Chef::Application.fatal!("Invalid MD5 checksum of #{downloaded_zookeeper_tarball}, expected: #{md5checksum}")
end
=begin
--- /dev/null
+name 'pig'
+description 'Pig application'
+
+run_list(
+ #'recipe[yum]',
+ 'role[java]',
+ 'recipe[grid]',
+ 'recipe[hadoop]',
+ 'recipe[pig]'
+)
+
+#env_run_lists "prod" => ["recipe[apache2]"], "staging" => ["recipe[apache2::staging]"], "_default" => []
+
+default_attributes(
+)
+
+#override_attributes "apache2" => { "max_children" => "50" }
'role[mocker]',
'role[hadoop-pseudo-distributed-with-security]',
'role[zookeeper-pseudo-replicated-with-security]',
- 'role[hbase-pseudo-distributed]'
+ 'role[hbase-pseudo-distributed]',
+ 'role[pig]'
)
#env_run_lists "prod" => ["recipe[apache2]"], "staging" => ["recipe[apache2::staging]"], "_default" => []
OSDN Git Service
