:name => 'admins',
:alias => 'AdministrativeStations',
:use_host_object => 'generic-admin-host',
+ :service_hostname => nil,
:nodes => 'adm00'.expand,
#:nodes => 'adm0{0..1}'.expand,
:mcast_addr => '239.2.10.0'
:name => 'nameservices',
:alias => 'NameServices',
:use_host_object => 'generic-nameservice-host',
+ :service_hostname => nil,
:nodes => 'ns00'.expand,
#:nodes => 'ns0{0..1}'.expand,
:mcast_addr => '239.2.10.1'
:name => 'nagioses',
:alias => 'Nagioses',
:use_host_object => 'generic-nagios-host',
+ :service_hostname => 'base-nm',
:nodes => 'nm90'.expand,
#:nodes => 'nm9{0..1}'.expand,
:mcast_addr => '239.2.10.2'
:name => 'gmetads',
:alias => 'Ganglia',
:use_host_object => 'generic-gmetad-host',
+ :service_hostname => 'gm90',
+ #:service_hostname => 'base-gm',
:nodes => 'gm90'.expand,
#:nodes => 'gm9{0..1}'.expand,
:mcast_addr => '239.2.10.3'
:name => 'zookeepers',
:alias => 'ZooKeepers',
:use_host_object => 'generic-zookeeper-host',
+ :service_hostname => nil,
:nodes => 'zk0{0..2}'.expand,
:mcast_addr => '239.2.10.4'
},
:name => 'provisioningmasters',
:alias => 'ProvisioningMasters',
:use_host_object => 'generic-provisioningmaster-host',
+ :service_hostname => 'base-gpm',
:nodes => 'gpm00'.expand,
#:nodes => 'gpm0{0..1}'.expand,
:mcast_addr => '239.2.10.5'
:name => 'nagioses',
:alias => 'Nagioses',
:use_host_object => 'generic-nagios-host',
+ :service_hostname => 'pleiades-nm',
:nodes => 'nm00'.expand,
#:nodes => 'nm0{0..1}'.expand,
:mcast_addr => '239.2.11.2'
:name => 'gmetads',
:alias => 'Ganglia',
:use_host_object => 'generic-gmetad-host',
+ :service_hostname => 'gm00',
+ #:service_hostname => 'pleiades-gm',
:nodes => 'gm00'.expand,
#:nodes => 'gm0{0..1}'.expand,
:mcast_addr => '239.2.11.3'
:name => 'metas',
:alias => 'Metas',
:use_host_object => 'generic-meta-host',
+ :service_hostname => 'pleiades-meta',
:nodes => 'meta00'.expand,
#:nodes => 'meta0{0..1}'.expand,
:mcast_addr => '239.2.11.4'
:name => 'gateways',
:alias => 'Gateways',
:use_host_object => 'generic-gateway-host',
+ :service_hostname => 'pleiades-gw',
:nodes => 'gw00'.expand,
#:nodes => 'gw0{0..1}'.expand,
:mcast_addr => '239.2.11.5'
:name => 'workflows',
:alias => 'Workflows',
:use_host_object => 'generic-workflow-host',
+ :service_hostname => 'pleiades-wf',
:nodes => 'wf00'.expand,
#:nodes => 'wf0{0..1}'.expand,
:mcast_addr => '239.2.11.6'
:name => 'proxies',
:alias => 'Proxies',
:use_host_object => 'generic-proxy-host',
+ :service_hostname => 'pleiades-pxy',
:nodes => 'pxy00'.expand,
#:nodes => 'pxy0{0..1}'.expand,
:mcast_addr => '239.2.11.7'
:name => 'namenodes',
:alias => 'NameNodes',
:use_host_object => 'generic-namenode-host',
+ :service_hostname => 'pleiades-nn',
:nodes => 'nn0{0,2}'.expand,
#:nodes => 'nn0{0..2}'.expand,
:mcast_addr => '239.2.11.8'
:name => 'resourcemanagers',
:alias => 'ResourceManagers',
:use_host_object => 'generic-resourcemanager-host',
+ :service_hostname => 'pleiades-rm',
:nodes => 'rm00'.expand,
#:nodes => 'rm0{0..1}'.expand,
:mcast_addr => '239.2.11.9'
:name => 'jobtrackers',
:alias => 'JobTrackers',
:use_host_object => 'generic-jobtracker-host',
+ :service_hostname => 'pleiades-jt',
:nodes => 'jt00'.expand,
#:nodes => 'jt0{0..1}'.expand,
:mcast_addr => '239.2.11.9'
:name => 'historyservers',
:alias => 'HistoryServers',
:use_host_object => 'generic-historyserver-host',
+ :service_hostname => 'pleiades-jh',
:nodes => 'jh0{0..1}'.expand,
:mcast_addr => '239.2.11.9'
},
:name => 'hbasemasters',
:alias => 'HBaseMasters',
:use_host_object => 'generic-hbasemaster-host',
+ :service_hostname => nil,
:nodes => 'hm0{0..1}'.expand,
:mcast_addr => '239.2.11.10'
},
:name => 'datanodes',
:alias => 'DataNodes',
:use_host_object => 'generic-datanode-host',
+ :service_hostname => nil,
:nodes => 'dn0000{0..1}'.expand,
:mcast_addr => '239.2.11.20'
}
--- /dev/null
+#
+# Cookbook Name:: hadoop
+# Library:: Helper
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Helper
+ def conf_template(conf_dir, middle_version, conf_files, tpl_vars)
+ source_dir = (middle_version == '2.0') \
+ ? "etc-#{middle_version}/hadoop" \
+ : "conf-#{middle_version}"
+
+ conf_files.each {|conf_file|
+ template "#{conf_dir}/#{conf_file}" do
+ source "#{source_dir}/#{conf_file}"
+ owner 'root'
+ group 'root'
+ mode '0644'
+ variables(tpl_vars)
+ end
+ }
+ end
+
+ def parse_version_number(version)
+ if /^(\d+)\.(\d+)\.(\d+)\.?(\d*)[-\w]*$/ =~ version then
+ major_version = $1
+ middle_version = "#{$1}.#{$2}"
+ else
+ Chef::Application.fatal!("Invalid Hadoop version: #{version}")
+ end
+
+ return [version, major_version, middle_version]
+ end
+end
--- /dev/null
+#
+# Cookbook Name:: hadoop
+# Recipe:: apache
+#
+# Copyright 2013, whitestar
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'digest/sha2'
+
+::Chef::Recipe.send(:include, Helper)
+
+users = nil
+limits_files = nil
+conf_files = nil
+
+users_v1 = {
+ :hadoop => {:name => 'hadoop', :uid => 10001},
+ :hdfs => {:name => 'hdfs', :uid => 10002},
+ :mapred => {:name => 'mapred', :uid => 10003}
+}
+users_v2 = users_v1.merge(
+ {:yarn => {:name => 'yarn', :uid => 10004}}
+)
+
+limits_files_v1 = [
+ 'hdfs.conf',
+ 'mapreduce.conf'
+]
+limits_files_v2 = limits_files_v1 + ['yarn.conf']
+
+conf_files_v1 = [
+ 'capacity-scheduler.xml',
+ 'configuration.xsl',
+ 'core-site.xml',
+ 'fair-scheduler.xml',
+ 'hadoop-env.sh',
+ 'hadoop-metrics2.properties',
+ 'hadoop-policy.xml',
+ 'hdfs-site.xml',
+ 'hosts.include',
+ 'hosts.exclude',
+ 'log4j.properties',
+ 'mapred-queue-acls.xml',
+ 'mapred-site.xml',
+ 'masters',
+ 'slaves',
+ #'ssl-client.xml',
+ #'ssl-server.xml'
+]
+
+conf_files_v2 = \
+ conf_files_v1 \
+ - [
+ 'fair-scheduler.xml',
+ 'mapred-queue-acls.xml',
+ 'masters'
+ ] \
+ + [
+ 'httpfs-env.sh',
+ 'httpfs-log4j.properties',
+ 'httpfs-signature.secret',
+ 'httpfs-site.xml',
+ 'mapred-env.sh',
+ 'yarn-env.sh',
+ 'yarn-site.xml'
+ ]
+
+# for ver. 1.0
+krb5_conf_files = [
+ 'krb5-strong.conf',
+ 'krb5-weak.conf'
+]
+
+version, major_version, middle_version \
+ = parse_version_number(node['hadoop']['version'])
+
+unless (('1.0.0' <= version && version < '1.2') \
+ || middle_version == '2.0') then
+ Chef::Application.fatal!("Non supported version: #{version}")
+end
+
+hadoop_members = nil
+case major_version
+ when '1'
+ users = users_v1
+ limits_files = limits_files_v1
+ conf_files = conf_files_v1
+ when '2'
+ users = users_v2
+ limits_files = limits_files_v2
+ conf_files = conf_files_v2
+end
+
+hadoop_members = []
+users.each {|key, user|
+ if key != :hadoop
+ hadoop_members.push(user[:name])
+
+ group user[:name] do
+ gid user[:uid]
+ members []
+ action :create
+ not_if "getent group #{user[:name]}"
+ end
+
+ user user[:name] do
+ uid user[:uid]
+ gid user[:uid]
+ home "/home/#{user[:name]}"
+ shell '/bin/sh'
+ password nil
+ supports :manage_home => false
+ not_if "getent passwd #{user[:name]}"
+ end
+ end
+}
+
+group users[:hadoop][:name] do
+ gid users[:hadoop][:uid]
+ members hadoop_members
+ append true
+ action :create
+ not_if "getent group #{users[:hadoop][:name]}"
+end
+
+user users[:hadoop][:name] do
+ uid users[:hadoop][:uid]
+ gid users[:hadoop][:uid]
+ home "/home/#{users[:hadoop][:name]}"
+ shell '/bin/sh'
+ password nil
+ supports :manage_home => false
+ not_if "getent passwd #{users[:hadoop][:name]}"
+end
+
+active_vol_nums = 0
+node['grid']['max_vol_nums'].to_i.times {|vol_num|
+ target_vol_dir = "#{node['grid']['vol_root']}/#{vol_num}"
+
+ if vol_num == 0 || FileTest::directory?(target_vol_dir) then
+ directory "#{target_vol_dir}/var" do
+ owner 'root'
+ group 'root'
+ mode '0755'
+ action :create
+ recursive true
+ end
+
+ %w{lib log}.each {|dir|
+ directory "#{target_vol_dir}/var/#{dir}" do
+ owner 'root'
+ group 'hadoop'
+ mode '0775'
+ action :create
+ recursive true
+ end
+ }
+
+ if major_version == '2' then
+ directory "#{target_vol_dir}/var/log/yarn" do
+ owner 'yarn'
+ group 'hadoop'
+ mode '0755'
+ action :create
+ recursive true
+ end
+
+ directory "#{target_vol_dir}/var/log/yarn/nm" do
+ owner 'yarn'
+ group 'hadoop'
+ mode '0755'
+ action :create
+ recursive true
+ end
+
+ directory "#{target_vol_dir}/var/log/yarn/nm/local" do
+ owner 'yarn'
+ group 'hadoop'
+ mode '0755'
+ action :create
+ recursive true
+ end
+ end
+
+ directory "#{target_vol_dir}/tmp" do
+ owner 'root'
+ group 'root'
+ mode '1777'
+ action :create
+ recursive true
+ end
+
+ if major_version == '2' then
+ directory "#{target_vol_dir}/mapred/tmp" do
+ owner 'root'
+ group 'root'
+ mode '1777'
+ action :create
+ recursive true
+ end
+ end
+
+ if vol_num == 0 then
+ directory "#{target_vol_dir}/var/run" do
+ owner 'root'
+ group 'hadoop'
+ mode '0775'
+ action :create
+ recursive true
+ end
+
+ directory "#{target_vol_dir}/var/log/hdfs" do
+ owner 'hdfs'
+ group 'hdfs'
+ mode '0755'
+ action :create
+ recursive true
+ end
+ end
+ else
+ break
+ end
+ active_vol_nums = vol_num + 1
+}
+log "This node active volumes: #{active_vol_nums}"
+
+file_cache_path = Chef::Config[:file_cache_path]
+install_root = "#{node['grid']['app_root']}/hadoop-#{version}"
+tarball = nil
+case major_version
+ when '1'
+ tarball = "hadoop-#{version}-bin.tar.gz"
+ when '2'
+ tarball = "hadoop-#{version}.tar.gz"
+end
+tarball_mds = "#{tarball}.mds"
+downloaded_tarball = "#{file_cache_path}/#{tarball}"
+downloaded_tarball_mds = "#{file_cache_path}/#{tarball_mds}"
+
+archive_url = node['hadoop']['archive_url']
+if ! FileTest.directory? install_root then
+ remote_file downloaded_tarball_mds do
+ source "#{archive_url}/hadoop-#{version}/#{tarball_mds}"
+ action :create_if_missing
+ end
+
+ remote_file downloaded_tarball do
+ source "#{archive_url}/hadoop-#{version}/#{tarball}"
+ action :create_if_missing
+ end
+
+ ruby_block "sha256 checksum #{downloaded_tarball}" do
+ block do
+=begin
+e.g. md file format
+'hadoop-1.1.2-bin.tar.gz: MD5 = 4B 59 F4 81 A7 52 D2 A9 20 3D D7 D0 A9 50 5C
+ 18
+hadoop-1.1.2-bin.tar.gz: SHA1 = DCCC 01A0 4C42 587D 9DF1 83CA 7DC8 83F7 A6A4
+ 8D80
+...'
+=end
+ checksum = File.read(downloaded_tarball_mds).
+ gsub(/(\s)+/, '').
+ scan(/#{tarball}:(.+?)=([0-9A-Z]+)/).
+ assoc('SHA256')[1]
+ Chef::Log.info "#{tarball}: SHA256 = #{checksum}"
+ actual_checksum = Digest::SHA256.file(downloaded_tarball).to_s
+ Chef::Log.info "#{tarball}: actual SHA256 = #{actual_checksum}"
+ if ! checksum.casecmp(actual_checksum) then
+ Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_tarball}, expected: #{checksum}")
+ end
+ end
+ action :create
+ end
+
+ pkg = 'tar'
+ resources(:package => pkg) rescue package pkg do
+ action :install
+ end
+
+ bash "install_hadoop-#{version}" do
+ code <<-EOC
+ tar xvzf #{downloaded_tarball} -C #{node['grid']['app_root']}
+ # for 2.0.x
+ chown -R root:root #{install_root}
+ EOC
+ creates install_root
+ end
+end
+
+link node['hadoop']['HADOOP_PREFIX'] do
+ to install_root
+ action [:delete, :create]
+end
+
+limits_files.each {|limits_file|
+ template "/etc/security/limits.d/#{limits_file}" do
+ source "etc/security/limits.d/#{limits_file}"
+ owner 'root'
+ group 'root'
+ mode '0644'
+ end
+}
+
+conf_dir = nil
+case major_version
+ when '1'
+ conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/conf"
+ when '2'
+ conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop"
+end
+
+tpl_vars = {
+ :active_vol_nums => active_vol_nums
+}
+conf_template(conf_dir, middle_version, conf_files, tpl_vars)
+
+# with security
+if node['hadoop']['with_security'] then
+ directory node['hadoop']['this.keytab.dir'] do
+ owner 'root'
+ group 'root'
+ mode '0755'
+ action :create
+ recursive true
+ end
+
+ jsvc_pkg = nil
+ case node[:platform_family]
+ when 'debian'
+ jsvc_pkg = 'jsvc'
+ when 'rhel'
+ jsvc_pkg = 'jakarta-commons-daemon-jsvc'
+ end
+
+ case major_version
+ when '1'
+ if node[:kernel][:machine] != 'x86_64' then
+ package jsvc_pkg do
+ action :install
+ end
+
+ link "#{install_root}/libexec/jsvc.i386" do
+ to '/usr/bin/jsvc'
+ end
+ end
+
+ file "#{node['grid']['app_root']}/hadoop-#{version}/bin/task-controller" do
+ owner 'root'
+ group 'mapred'
+ mode '6050'
+ end
+
+ template "#{node['grid']['app_root']}/hadoop-#{version}/conf/taskcontroller.cfg" do
+ source "conf-#{middle_version}/taskcontroller.cfg"
+ owner 'root'
+ group 'root'
+ mode '0400'
+ variables({
+ :active_vol_nums => active_vol_nums
+ })
+ end
+ when '2'
+ package jsvc_pkg do
+ action :install
+ end
+
+ file "#{node['grid']['app_root']}/hadoop-#{version}/bin/container-executor" do
+ owner 'root'
+ group 'yarn'
+ mode '6050'
+ end
+
+ template "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop/container-executor.cfg" do
+ source "etc-#{middle_version}/hadoop/container-executor.cfg"
+ owner 'root'
+ group 'root'
+ mode '0400'
+ variables({
+ :active_vol_nums => active_vol_nums
+ })
+ end
+ end
+
+ if middle_version == '1.0' then
+ tpl_vars = nil
+ conf_template(conf_dir, middle_version, krb5_conf_files, tpl_vars)
+ end
+end
+
+case major_version
+ when '1'
+ log <<-EOM
+Note:
+You must initialize HDFS in the first installation:
+ $ cd #{node['grid']['app_root']}/hadoop
+ $ sudo -u hdfs ./bin/hadoop namenode -format
+ $ sudo -u hdfs ./bin/hadoop-daemon.sh start namenode
+ $ sudo -u hdfs ./bin/hadoop-daemon.sh start datanode
+ $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs /
+ $ sudo -u hdfs ./bin/hadoop fs -chmod 755 /
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir /user
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred
+ $ sudo -u hdfs ./bin/hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred
+ EOM
+
+ examples_jar = "hadoop-examples-#{version}.jar"
+ when '2'
+ log <<-EOM
+Note:
+You must initialize HDFS in the first installation:
+ $ cd #{node['grid']['app_root']}/hadoop
+ $ sudo -u hdfs ./bin/hdfs namenode -format
+ $ sudo -u hdfs ./sbin/hadoop-daemon.sh start namenode
+ $ sudo -u hdfs ./sbin/hadoop-daemon.sh start datanode
+ $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs /
+ $ sudo -u hdfs ./bin/hadoop fs -chmod 755 /
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir /user
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm
+ $ sudo -u hdfs ./bin/hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm
+ $ sudo -u hdfs ./bin/hadoop fs -chmod 1777 #{node['grid']['vol_root']}/0/var/log/yarn/nm
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history
+ $ sudo -u hdfs ./bin/hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred
+ $ sudo -u hdfs ./bin/hadoop fs -chmod -R 755 #{node['grid']['vol_root']}/0/var/lib/mapred
+ EOM
+
+ examples_jar = "share/hadoop/mapreduce/hadoop-mapreduce-examples-#{version}.jar"
+end
+
+if node['hadoop']['with_security'] then
+ log <<-EOM
+Note:
+Example MapReduce job execution:
+ $ sudo -u alice kinit
+ Password for alice@LOCALDOMAIN:
+ $ sudo -u alice bin/hadoop jar #{examples_jar} pi \\
+ > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10
+ EOM
+else
+ log <<-EOM
+Note:
+Example MapReduce job execution:
+ $ sudo adduser alice
+ $ sudo -u hdfs ./bin/hadoop fs -mkdir /user/alice
+ $ sudo -u hdfs ./bin/hadoop fs -chown alice:alice /user/alice
+ $ sudo -u alice ./bin/hadoop jar #{examples_jar} pi 5 10
+ EOM
+end
+
# limitations under the License.
#
-require 'digest/sha2'
-
-users = nil
-limits_files = nil
-conf_files = nil
-
-users_v1 = {
- :hadoop => {:name => 'hadoop', :uid => 10001},
- :hdfs => {:name => 'hdfs', :uid => 10002},
- :mapred => {:name => 'mapred', :uid => 10003}
-}
-users_v2 = users_v1.merge(
- {:yarn => {:name => 'yarn', :uid => 10004}}
-)
-
-limits_files_v1 = [
- 'hdfs.conf',
- 'mapreduce.conf'
-]
-limits_files_v2 = limits_files_v1 + ['yarn.conf']
-
-conf_files_v1 = [
- 'capacity-scheduler.xml',
- 'configuration.xsl',
- 'core-site.xml',
- 'fair-scheduler.xml',
- 'hadoop-env.sh',
- 'hadoop-metrics2.properties',
- 'hadoop-policy.xml',
- 'hdfs-site.xml',
- 'hosts.include',
- 'hosts.exclude',
- 'log4j.properties',
- 'mapred-queue-acls.xml',
- 'mapred-site.xml',
- 'masters',
- 'slaves',
- #'ssl-client.xml',
- #'ssl-server.xml'
-]
-
-conf_files_v2 = \
- conf_files_v1 \
- - [
- 'fair-scheduler.xml',
- 'mapred-queue-acls.xml',
- 'masters'
- ] \
- + [
- 'httpfs-env.sh',
- 'httpfs-log4j.properties',
- 'httpfs-signature.secret',
- 'httpfs-site.xml',
- 'mapred-env.sh',
- 'yarn-env.sh',
- 'yarn-site.xml'
- ]
-
-# for ver. 1.0
-krb5_conf_files = [
- 'krb5-strong.conf',
- 'krb5-weak.conf'
-]
-
-def conf_template(conf_dir, middle_version, conf_files, tpl_vars)
- source_dir = (middle_version == '2.0') \
- ? "etc-#{middle_version}/hadoop" \
- : "conf-#{middle_version}"
-
- conf_files.each {|conf_file|
- template "#{conf_dir}/#{conf_file}" do
- source "#{source_dir}/#{conf_file}"
- owner 'root'
- group 'root'
- mode '0644'
- variables(tpl_vars)
- end
- }
-end
-
-version = node['hadoop']['version']
-major_version = nil
-middle_version = nil
-
-if /^(\d+)\.(\d+)\.(\d+)\.?(\d*)[-\w]*$/ =~ version then
- major_version = $1
- middle_version = "#{$1}.#{$2}"
-else
- Chef::Application.fatal!("Invalid Hadoop version: #{version}")
-end
-
-unless (('1.0.0' <= version && version < '1.2') \
- || middle_version == '2.0') then
- Chef::Application.fatal!("Non supported version: #{version}")
-end
-
-hadoop_members = nil
-case major_version
- when '1'
- users = users_v1
- limits_files = limits_files_v1
- conf_files = conf_files_v1
- when '2'
- users = users_v2
- limits_files = limits_files_v2
- conf_files = conf_files_v2
-end
-
-hadoop_members = []
-users.each {|key, user|
- if key != :hadoop
- hadoop_members.push(user[:name])
-
- group user[:name] do
- gid user[:uid]
- members []
- action :create
- not_if "getent group #{user[:name]}"
- end
-
- user user[:name] do
- uid user[:uid]
- gid user[:uid]
- home "/home/#{user[:name]}"
- shell '/bin/sh'
- password nil
- supports :manage_home => false
- not_if "getent passwd #{user[:name]}"
- end
- end
-}
-
-group users[:hadoop][:name] do
- gid users[:hadoop][:uid]
- members hadoop_members
- append true
- action :create
- not_if "getent group #{users[:hadoop][:name]}"
-end
-
-user users[:hadoop][:name] do
- uid users[:hadoop][:uid]
- gid users[:hadoop][:uid]
- home "/home/#{users[:hadoop][:name]}"
- shell '/bin/sh'
- password nil
- supports :manage_home => false
- not_if "getent passwd #{users[:hadoop][:name]}"
-end
-
-active_vol_nums = 0
-node['grid']['max_vol_nums'].to_i.times {|vol_num|
- target_vol_dir = "#{node['grid']['vol_root']}/#{vol_num}"
-
- if vol_num == 0 || FileTest::directory?(target_vol_dir) then
- directory "#{target_vol_dir}/var" do
- owner 'root'
- group 'root'
- mode '0755'
- action :create
- recursive true
- end
-
- %w{lib log}.each {|dir|
- directory "#{target_vol_dir}/var/#{dir}" do
- owner 'root'
- group 'hadoop'
- mode '0775'
- action :create
- recursive true
- end
- }
-
- if major_version == '2' then
- directory "#{target_vol_dir}/var/log/yarn" do
- owner 'yarn'
- group 'hadoop'
- mode '0755'
- action :create
- recursive true
- end
-
- directory "#{target_vol_dir}/var/log/yarn/nm" do
- owner 'yarn'
- group 'hadoop'
- mode '0755'
- action :create
- recursive true
- end
-
- directory "#{target_vol_dir}/var/log/yarn/nm/local" do
- owner 'yarn'
- group 'hadoop'
- mode '0755'
- action :create
- recursive true
- end
- end
-
- directory "#{target_vol_dir}/tmp" do
- owner 'root'
- group 'root'
- mode '1777'
- action :create
- recursive true
- end
-
- if major_version == '2' then
- directory "#{target_vol_dir}/mapred/tmp" do
- owner 'root'
- group 'root'
- mode '1777'
- action :create
- recursive true
- end
- end
-
- if vol_num == 0 then
- directory "#{target_vol_dir}/var/run" do
- owner 'root'
- group 'hadoop'
- mode '0775'
- action :create
- recursive true
- end
-
- directory "#{target_vol_dir}/var/log/hdfs" do
- owner 'hdfs'
- group 'hdfs'
- mode '0755'
- action :create
- recursive true
- end
- end
+flavor = node['hadoop']['install_flavor']
+case flavor
+ when 'apache'
+ include_recipe "hadoop::#{flavor}"
else
- break
- end
- active_vol_nums = vol_num + 1
-}
-log "This node active volumes: #{active_vol_nums}"
-
-file_cache_path = Chef::Config[:file_cache_path]
-install_root = "#{node['grid']['app_root']}/hadoop-#{version}"
-tarball = nil
-case major_version
- when '1'
- tarball = "hadoop-#{version}-bin.tar.gz"
- when '2'
- tarball = "hadoop-#{version}.tar.gz"
-end
-tarball_mds = "#{tarball}.mds"
-downloaded_tarball = "#{file_cache_path}/#{tarball}"
-downloaded_tarball_mds = "#{file_cache_path}/#{tarball_mds}"
-
-archive_url = node['hadoop']['archive_url']
-if ! FileTest.directory? install_root then
- remote_file downloaded_tarball_mds do
- source "#{archive_url}/hadoop-#{version}/#{tarball_mds}"
- action :create_if_missing
- end
-
- remote_file downloaded_tarball do
- source "#{archive_url}/hadoop-#{version}/#{tarball}"
- action :create_if_missing
- end
-
- ruby_block "sha256 checksum #{downloaded_tarball}" do
- block do
-=begin
-e.g. md file format
-'hadoop-1.1.2-bin.tar.gz: MD5 = 4B 59 F4 81 A7 52 D2 A9 20 3D D7 D0 A9 50 5C
- 18
-hadoop-1.1.2-bin.tar.gz: SHA1 = DCCC 01A0 4C42 587D 9DF1 83CA 7DC8 83F7 A6A4
- 8D80
-...'
-=end
- checksum = File.read(downloaded_tarball_mds).
- gsub(/(\s)+/, '').
- scan(/#{tarball}:(.+?)=([0-9A-Z]+)/).
- assoc('SHA256')[1]
- Chef::Log.info "#{tarball}: SHA256 = #{checksum}"
- actual_checksum = Digest::SHA256.file(downloaded_tarball).to_s
- Chef::Log.info "#{tarball}: actual SHA256 = #{actual_checksum}"
- if ! checksum.casecmp(actual_checksum) then
- Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_tarball}, expected: #{checksum}")
- end
- end
- action :create
- end
-
- pkg = 'tar'
- resources(:package => pkg) rescue package pkg do
- action :install
- end
-
- bash "install_hadoop-#{version}" do
- code <<-EOC
- tar xvzf #{downloaded_tarball} -C #{node['grid']['app_root']}
- # for 2.0.x
- chown -R root:root #{install_root}
- EOC
- creates install_root
- end
-end
-
-link node['hadoop']['HADOOP_PREFIX'] do
- to install_root
- action [:delete, :create]
-end
-
-limits_files.each {|limits_file|
- template "/etc/security/limits.d/#{limits_file}" do
- source "etc/security/limits.d/#{limits_file}"
- owner 'root'
- group 'root'
- mode '0644'
- end
-}
-
-conf_dir = nil
-case major_version
- when '1'
- conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/conf"
- when '2'
- conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop"
-end
-
-tpl_vars = {
- :active_vol_nums => active_vol_nums
-}
-conf_template(conf_dir, middle_version, conf_files, tpl_vars)
-
-# with security
-if node['hadoop']['with_security'] then
- directory node['hadoop']['this.keytab.dir'] do
- owner 'root'
- group 'root'
- mode '0755'
- action :create
- recursive true
- end
-
- jsvc_pkg = nil
- case node[:platform_family]
- when 'debian'
- jsvc_pkg = 'jsvc'
- when 'rhel'
- jsvc_pkg = 'jakarta-commons-daemon-jsvc'
- end
-
- case major_version
- when '1'
- if node[:kernel][:machine] != 'x86_64' then
- package jsvc_pkg do
- action :install
- end
-
- link "#{install_root}/libexec/jsvc.i386" do
- to '/usr/bin/jsvc'
- end
- end
-
- file "#{node['grid']['app_root']}/hadoop-#{version}/bin/task-controller" do
- owner 'root'
- group 'mapred'
- mode '6050'
- end
-
- template "#{node['grid']['app_root']}/hadoop-#{version}/conf/taskcontroller.cfg" do
- source "conf-#{middle_version}/taskcontroller.cfg"
- owner 'root'
- group 'root'
- mode '0400'
- variables({
- :active_vol_nums => active_vol_nums
- })
- end
- when '2'
- package jsvc_pkg do
- action :install
- end
-
- file "#{node['grid']['app_root']}/hadoop-#{version}/bin/container-executor" do
- owner 'root'
- group 'yarn'
- mode '6050'
- end
-
- template "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop/container-executor.cfg" do
- source "etc-#{middle_version}/hadoop/container-executor.cfg"
- owner 'root'
- group 'root'
- mode '0400'
- variables({
- :active_vol_nums => active_vol_nums
- })
- end
- end
-
- if middle_version == '1.0' then
- tpl_vars = nil
- conf_template(conf_dir, middle_version, krb5_conf_files, tpl_vars)
- end
-end
-
-case major_version
- when '1'
- log <<-EOM
-Note:
-You must initialize HDFS in the first installation:
- $ cd #{node['grid']['app_root']}/hadoop
- $ sudo -u hdfs ./bin/hadoop namenode -format
- $ sudo -u hdfs ./bin/hadoop-daemon.sh start namenode
- $ sudo -u hdfs ./bin/hadoop-daemon.sh start datanode
- $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs /
- $ sudo -u hdfs ./bin/hadoop fs -chmod 755 /
- $ sudo -u hdfs ./bin/hadoop fs -mkdir /user
- $ sudo -u hdfs ./bin/hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred
- $ sudo -u hdfs ./bin/hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred
- EOM
-
- examples_jar = "hadoop-examples-#{version}.jar"
- when '2'
- log <<-EOM
-Note:
-You must initialize HDFS in the first installation:
- $ cd #{node['grid']['app_root']}/hadoop
- $ sudo -u hdfs ./bin/hdfs namenode -format
- $ sudo -u hdfs ./sbin/hadoop-daemon.sh start namenode
- $ sudo -u hdfs ./sbin/hadoop-daemon.sh start datanode
- $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs /
- $ sudo -u hdfs ./bin/hadoop fs -chmod 755 /
- $ sudo -u hdfs ./bin/hadoop fs -mkdir /user
- $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm
- $ sudo -u hdfs ./bin/hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm
- $ sudo -u hdfs ./bin/hadoop fs -chmod 1777 #{node['grid']['vol_root']}/0/var/log/yarn/nm
- $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history
- $ sudo -u hdfs ./bin/hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred
- $ sudo -u hdfs ./bin/hadoop fs -chmod -R 755 #{node['grid']['vol_root']}/0/var/lib/mapred
- EOM
-
- examples_jar = "share/hadoop/mapreduce/hadoop-mapreduce-examples-#{version}.jar"
-end
-
-if node['hadoop']['with_security'] then
- log <<-EOM
-Note:
-Example MapReduce job execution:
- $ sudo -u alice kinit
- Password for alice@LOCALDOMAIN:
- $ sudo -u alice bin/hadoop jar #{examples_jar} pi \\
- > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10
- EOM
-else
- log <<-EOM
-Note:
-Example MapReduce job execution:
- $ sudo adduser alice
- $ sudo -u hdfs ./bin/hadoop fs -mkdir /user/alice
- $ sudo -u hdfs ./bin/hadoop fs -chown alice:alice /user/alice
- $ sudo -u alice ./bin/hadoop jar #{examples_jar} pi 5 10
- EOM
+ Chef::Application.fatal!("Non supported flavor: #{flavor}")
end
'authorized_for_all_host_commands' => '*',
'check_ganglia_metric' => {
'enabled' => true,
- 'gmetad_host' => "gm90.#{Grid::DOMAIN}"
+ 'gmetad_host' => "#{this_cluster[:gmetads][:service_hostname]}.#{Grid::DOMAIN}"
},
}
)
{
'host_name' => 'localhost',
'service_description' => 'Cache for check_ganglia_metric',
- 'dependent_host_name' => '.*',
- #'dependent_hostgroup_name' => 'grid-servers',
+ #'dependent_host_name' => '.*', # NG!
+ 'dependent_hostgroup_name' => 'grid-servers',
'dependent_service_description' => '.* via Ganglia$',
'execution_failure_criteria' => 'c,p'
}
'nagios' => {
'objects' => nagios_objects,
'web' => {
- 'Allow' => ["from .#{Grid::DOMAIN}"],
+ 'Allow' => [
+ "from .#{Grid::DOMAIN}",
+ 'from 192.168.0.0/16'],
'AuthType' => 'Kerberos',
'mod_auth_kerb' => {
'KrbAuthRealms' => Grid::REALM,
'authorized_for_all_host_commands' => '*',
'check_ganglia_metric' => {
'enabled' => true,
- 'gmetad_host' => "gm00.#{Grid::DOMAIN}"
+ 'gmetad_host' => "#{this_cluster[:gmetads][:service_hostname]}.#{Grid::DOMAIN}"
},
}
)
OSDN Git Service
