From: whitestar Date: Tue, 23 Jul 2013 10:37:07 +0000 (+0900) Subject: refactor hadoop recipe. X-Git-Tag: node_commons-0.1.1~84 X-Git-Url: http://git.osdn.net/view?a=commitdiff_plain;h=47039df8c40454fabf1aa20ba56f6a1584cb4dd6;p=metasearch%2Fgrid-chef-repo.git refactor hadoop recipe. --- diff --git a/_lib/grid.rb b/_lib/grid.rb index 4977723..08b047b 100644 --- a/_lib/grid.rb +++ b/_lib/grid.rb @@ -13,6 +13,7 @@ module Grid :name => 'admins', :alias => 'AdministrativeStations', :use_host_object => 'generic-admin-host', + :service_hostname => nil, :nodes => 'adm00'.expand, #:nodes => 'adm0{0..1}'.expand, :mcast_addr => '239.2.10.0' @@ -21,6 +22,7 @@ module Grid :name => 'nameservices', :alias => 'NameServices', :use_host_object => 'generic-nameservice-host', + :service_hostname => nil, :nodes => 'ns00'.expand, #:nodes => 'ns0{0..1}'.expand, :mcast_addr => '239.2.10.1' @@ -29,6 +31,7 @@ module Grid :name => 'nagioses', :alias => 'Nagioses', :use_host_object => 'generic-nagios-host', + :service_hostname => 'base-nm', :nodes => 'nm90'.expand, #:nodes => 'nm9{0..1}'.expand, :mcast_addr => '239.2.10.2' @@ -37,6 +40,8 @@ module Grid :name => 'gmetads', :alias => 'Ganglia', :use_host_object => 'generic-gmetad-host', + :service_hostname => 'gm90', + #:service_hostname => 'base-gm', :nodes => 'gm90'.expand, #:nodes => 'gm9{0..1}'.expand, :mcast_addr => '239.2.10.3' @@ -45,6 +50,7 @@ module Grid :name => 'zookeepers', :alias => 'ZooKeepers', :use_host_object => 'generic-zookeeper-host', + :service_hostname => nil, :nodes => 'zk0{0..2}'.expand, :mcast_addr => '239.2.10.4' }, @@ -52,6 +58,7 @@ module Grid :name => 'provisioningmasters', :alias => 'ProvisioningMasters', :use_host_object => 'generic-provisioningmaster-host', + :service_hostname => 'base-gpm', :nodes => 'gpm00'.expand, #:nodes => 'gpm0{0..1}'.expand, :mcast_addr => '239.2.10.5' @@ -62,6 +69,7 @@ module Grid :name => 'nagioses', :alias => 'Nagioses', :use_host_object => 'generic-nagios-host', + :service_hostname => 'pleiades-nm', :nodes => 'nm00'.expand, #:nodes => 'nm0{0..1}'.expand, :mcast_addr => '239.2.11.2' @@ -70,6 +78,8 @@ module Grid :name => 'gmetads', :alias => 'Ganglia', :use_host_object => 'generic-gmetad-host', + :service_hostname => 'gm00', + #:service_hostname => 'pleiades-gm', :nodes => 'gm00'.expand, #:nodes => 'gm0{0..1}'.expand, :mcast_addr => '239.2.11.3' @@ -79,6 +89,7 @@ module Grid :name => 'metas', :alias => 'Metas', :use_host_object => 'generic-meta-host', + :service_hostname => 'pleiades-meta', :nodes => 'meta00'.expand, #:nodes => 'meta0{0..1}'.expand, :mcast_addr => '239.2.11.4' @@ -88,6 +99,7 @@ module Grid :name => 'gateways', :alias => 'Gateways', :use_host_object => 'generic-gateway-host', + :service_hostname => 'pleiades-gw', :nodes => 'gw00'.expand, #:nodes => 'gw0{0..1}'.expand, :mcast_addr => '239.2.11.5' @@ -96,6 +108,7 @@ module Grid :name => 'workflows', :alias => 'Workflows', :use_host_object => 'generic-workflow-host', + :service_hostname => 'pleiades-wf', :nodes => 'wf00'.expand, #:nodes => 'wf0{0..1}'.expand, :mcast_addr => '239.2.11.6' @@ -104,6 +117,7 @@ module Grid :name => 'proxies', :alias => 'Proxies', :use_host_object => 'generic-proxy-host', + :service_hostname => 'pleiades-pxy', :nodes => 'pxy00'.expand, #:nodes => 'pxy0{0..1}'.expand, :mcast_addr => '239.2.11.7' @@ -112,6 +126,7 @@ module Grid :name => 'namenodes', :alias => 'NameNodes', :use_host_object => 'generic-namenode-host', + :service_hostname => 'pleiades-nn', :nodes => 'nn0{0,2}'.expand, #:nodes => 'nn0{0..2}'.expand, :mcast_addr => '239.2.11.8' @@ -121,6 +136,7 @@ module Grid :name => 'resourcemanagers', :alias => 'ResourceManagers', :use_host_object => 'generic-resourcemanager-host', + :service_hostname => 'pleiades-rm', :nodes => 'rm00'.expand, #:nodes => 'rm0{0..1}'.expand, :mcast_addr => '239.2.11.9' @@ -130,6 +146,7 @@ module Grid :name => 'jobtrackers', :alias => 'JobTrackers', :use_host_object => 'generic-jobtracker-host', + :service_hostname => 'pleiades-jt', :nodes => 'jt00'.expand, #:nodes => 'jt0{0..1}'.expand, :mcast_addr => '239.2.11.9' @@ -139,6 +156,7 @@ module Grid :name => 'historyservers', :alias => 'HistoryServers', :use_host_object => 'generic-historyserver-host', + :service_hostname => 'pleiades-jh', :nodes => 'jh0{0..1}'.expand, :mcast_addr => '239.2.11.9' }, @@ -147,6 +165,7 @@ module Grid :name => 'hbasemasters', :alias => 'HBaseMasters', :use_host_object => 'generic-hbasemaster-host', + :service_hostname => nil, :nodes => 'hm0{0..1}'.expand, :mcast_addr => '239.2.11.10' }, @@ -154,6 +173,7 @@ module Grid :name => 'datanodes', :alias => 'DataNodes', :use_host_object => 'generic-datanode-host', + :service_hostname => nil, :nodes => 'dn0000{0..1}'.expand, :mcast_addr => '239.2.11.20' } diff --git a/cookbooks/hadoop/libraries/helper.rb b/cookbooks/hadoop/libraries/helper.rb new file mode 100644 index 0000000..ad87890 --- /dev/null +++ b/cookbooks/hadoop/libraries/helper.rb @@ -0,0 +1,47 @@ +# +# Cookbook Name:: hadoop +# Library:: Helper +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Helper + def conf_template(conf_dir, middle_version, conf_files, tpl_vars) + source_dir = (middle_version == '2.0') \ + ? "etc-#{middle_version}/hadoop" \ + : "conf-#{middle_version}" + + conf_files.each {|conf_file| + template "#{conf_dir}/#{conf_file}" do + source "#{source_dir}/#{conf_file}" + owner 'root' + group 'root' + mode '0644' + variables(tpl_vars) + end + } + end + + def parse_version_number(version) + if /^(\d+)\.(\d+)\.(\d+)\.?(\d*)[-\w]*$/ =~ version then + major_version = $1 + middle_version = "#{$1}.#{$2}" + else + Chef::Application.fatal!("Invalid Hadoop version: #{version}") + end + + return [version, major_version, middle_version] + end +end diff --git a/cookbooks/hadoop/recipes/apache.rb b/cookbooks/hadoop/recipes/apache.rb new file mode 100644 index 0000000..334bd63 --- /dev/null +++ b/cookbooks/hadoop/recipes/apache.rb @@ -0,0 +1,461 @@ +# +# Cookbook Name:: hadoop +# Recipe:: apache +# +# Copyright 2013, whitestar +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'digest/sha2' + +::Chef::Recipe.send(:include, Helper) + +users = nil +limits_files = nil +conf_files = nil + +users_v1 = { + :hadoop => {:name => 'hadoop', :uid => 10001}, + :hdfs => {:name => 'hdfs', :uid => 10002}, + :mapred => {:name => 'mapred', :uid => 10003} +} +users_v2 = users_v1.merge( + {:yarn => {:name => 'yarn', :uid => 10004}} +) + +limits_files_v1 = [ + 'hdfs.conf', + 'mapreduce.conf' +] +limits_files_v2 = limits_files_v1 + ['yarn.conf'] + +conf_files_v1 = [ + 'capacity-scheduler.xml', + 'configuration.xsl', + 'core-site.xml', + 'fair-scheduler.xml', + 'hadoop-env.sh', + 'hadoop-metrics2.properties', + 'hadoop-policy.xml', + 'hdfs-site.xml', + 'hosts.include', + 'hosts.exclude', + 'log4j.properties', + 'mapred-queue-acls.xml', + 'mapred-site.xml', + 'masters', + 'slaves', + #'ssl-client.xml', + #'ssl-server.xml' +] + +conf_files_v2 = \ + conf_files_v1 \ + - [ + 'fair-scheduler.xml', + 'mapred-queue-acls.xml', + 'masters' + ] \ + + [ + 'httpfs-env.sh', + 'httpfs-log4j.properties', + 'httpfs-signature.secret', + 'httpfs-site.xml', + 'mapred-env.sh', + 'yarn-env.sh', + 'yarn-site.xml' + ] + +# for ver. 1.0 +krb5_conf_files = [ + 'krb5-strong.conf', + 'krb5-weak.conf' +] + +version, major_version, middle_version \ + = parse_version_number(node['hadoop']['version']) + +unless (('1.0.0' <= version && version < '1.2') \ + || middle_version == '2.0') then + Chef::Application.fatal!("Non supported version: #{version}") +end + +hadoop_members = nil +case major_version + when '1' + users = users_v1 + limits_files = limits_files_v1 + conf_files = conf_files_v1 + when '2' + users = users_v2 + limits_files = limits_files_v2 + conf_files = conf_files_v2 +end + +hadoop_members = [] +users.each {|key, user| + if key != :hadoop + hadoop_members.push(user[:name]) + + group user[:name] do + gid user[:uid] + members [] + action :create + not_if "getent group #{user[:name]}" + end + + user user[:name] do + uid user[:uid] + gid user[:uid] + home "/home/#{user[:name]}" + shell '/bin/sh' + password nil + supports :manage_home => false + not_if "getent passwd #{user[:name]}" + end + end +} + +group users[:hadoop][:name] do + gid users[:hadoop][:uid] + members hadoop_members + append true + action :create + not_if "getent group #{users[:hadoop][:name]}" +end + +user users[:hadoop][:name] do + uid users[:hadoop][:uid] + gid users[:hadoop][:uid] + home "/home/#{users[:hadoop][:name]}" + shell '/bin/sh' + password nil + supports :manage_home => false + not_if "getent passwd #{users[:hadoop][:name]}" +end + +active_vol_nums = 0 +node['grid']['max_vol_nums'].to_i.times {|vol_num| + target_vol_dir = "#{node['grid']['vol_root']}/#{vol_num}" + + if vol_num == 0 || FileTest::directory?(target_vol_dir) then + directory "#{target_vol_dir}/var" do + owner 'root' + group 'root' + mode '0755' + action :create + recursive true + end + + %w{lib log}.each {|dir| + directory "#{target_vol_dir}/var/#{dir}" do + owner 'root' + group 'hadoop' + mode '0775' + action :create + recursive true + end + } + + if major_version == '2' then + directory "#{target_vol_dir}/var/log/yarn" do + owner 'yarn' + group 'hadoop' + mode '0755' + action :create + recursive true + end + + directory "#{target_vol_dir}/var/log/yarn/nm" do + owner 'yarn' + group 'hadoop' + mode '0755' + action :create + recursive true + end + + directory "#{target_vol_dir}/var/log/yarn/nm/local" do + owner 'yarn' + group 'hadoop' + mode '0755' + action :create + recursive true + end + end + + directory "#{target_vol_dir}/tmp" do + owner 'root' + group 'root' + mode '1777' + action :create + recursive true + end + + if major_version == '2' then + directory "#{target_vol_dir}/mapred/tmp" do + owner 'root' + group 'root' + mode '1777' + action :create + recursive true + end + end + + if vol_num == 0 then + directory "#{target_vol_dir}/var/run" do + owner 'root' + group 'hadoop' + mode '0775' + action :create + recursive true + end + + directory "#{target_vol_dir}/var/log/hdfs" do + owner 'hdfs' + group 'hdfs' + mode '0755' + action :create + recursive true + end + end + else + break + end + active_vol_nums = vol_num + 1 +} +log "This node active volumes: #{active_vol_nums}" + +file_cache_path = Chef::Config[:file_cache_path] +install_root = "#{node['grid']['app_root']}/hadoop-#{version}" +tarball = nil +case major_version + when '1' + tarball = "hadoop-#{version}-bin.tar.gz" + when '2' + tarball = "hadoop-#{version}.tar.gz" +end +tarball_mds = "#{tarball}.mds" +downloaded_tarball = "#{file_cache_path}/#{tarball}" +downloaded_tarball_mds = "#{file_cache_path}/#{tarball_mds}" + +archive_url = node['hadoop']['archive_url'] +if ! FileTest.directory? install_root then + remote_file downloaded_tarball_mds do + source "#{archive_url}/hadoop-#{version}/#{tarball_mds}" + action :create_if_missing + end + + remote_file downloaded_tarball do + source "#{archive_url}/hadoop-#{version}/#{tarball}" + action :create_if_missing + end + + ruby_block "sha256 checksum #{downloaded_tarball}" do + block do +=begin +e.g. md file format +'hadoop-1.1.2-bin.tar.gz: MD5 = 4B 59 F4 81 A7 52 D2 A9 20 3D D7 D0 A9 50 5C + 18 +hadoop-1.1.2-bin.tar.gz: SHA1 = DCCC 01A0 4C42 587D 9DF1 83CA 7DC8 83F7 A6A4 + 8D80 +...' +=end + checksum = File.read(downloaded_tarball_mds). + gsub(/(\s)+/, ''). + scan(/#{tarball}:(.+?)=([0-9A-Z]+)/). + assoc('SHA256')[1] + Chef::Log.info "#{tarball}: SHA256 = #{checksum}" + actual_checksum = Digest::SHA256.file(downloaded_tarball).to_s + Chef::Log.info "#{tarball}: actual SHA256 = #{actual_checksum}" + if ! checksum.casecmp(actual_checksum) then + Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_tarball}, expected: #{checksum}") + end + end + action :create + end + + pkg = 'tar' + resources(:package => pkg) rescue package pkg do + action :install + end + + bash "install_hadoop-#{version}" do + code <<-EOC + tar xvzf #{downloaded_tarball} -C #{node['grid']['app_root']} + # for 2.0.x + chown -R root:root #{install_root} + EOC + creates install_root + end +end + +link node['hadoop']['HADOOP_PREFIX'] do + to install_root + action [:delete, :create] +end + +limits_files.each {|limits_file| + template "/etc/security/limits.d/#{limits_file}" do + source "etc/security/limits.d/#{limits_file}" + owner 'root' + group 'root' + mode '0644' + end +} + +conf_dir = nil +case major_version + when '1' + conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/conf" + when '2' + conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop" +end + +tpl_vars = { + :active_vol_nums => active_vol_nums +} +conf_template(conf_dir, middle_version, conf_files, tpl_vars) + +# with security +if node['hadoop']['with_security'] then + directory node['hadoop']['this.keytab.dir'] do + owner 'root' + group 'root' + mode '0755' + action :create + recursive true + end + + jsvc_pkg = nil + case node[:platform_family] + when 'debian' + jsvc_pkg = 'jsvc' + when 'rhel' + jsvc_pkg = 'jakarta-commons-daemon-jsvc' + end + + case major_version + when '1' + if node[:kernel][:machine] != 'x86_64' then + package jsvc_pkg do + action :install + end + + link "#{install_root}/libexec/jsvc.i386" do + to '/usr/bin/jsvc' + end + end + + file "#{node['grid']['app_root']}/hadoop-#{version}/bin/task-controller" do + owner 'root' + group 'mapred' + mode '6050' + end + + template "#{node['grid']['app_root']}/hadoop-#{version}/conf/taskcontroller.cfg" do + source "conf-#{middle_version}/taskcontroller.cfg" + owner 'root' + group 'root' + mode '0400' + variables({ + :active_vol_nums => active_vol_nums + }) + end + when '2' + package jsvc_pkg do + action :install + end + + file "#{node['grid']['app_root']}/hadoop-#{version}/bin/container-executor" do + owner 'root' + group 'yarn' + mode '6050' + end + + template "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop/container-executor.cfg" do + source "etc-#{middle_version}/hadoop/container-executor.cfg" + owner 'root' + group 'root' + mode '0400' + variables({ + :active_vol_nums => active_vol_nums + }) + end + end + + if middle_version == '1.0' then + tpl_vars = nil + conf_template(conf_dir, middle_version, krb5_conf_files, tpl_vars) + end +end + +case major_version + when '1' + log <<-EOM +Note: +You must initialize HDFS in the first installation: + $ cd #{node['grid']['app_root']}/hadoop + $ sudo -u hdfs ./bin/hadoop namenode -format + $ sudo -u hdfs ./bin/hadoop-daemon.sh start namenode + $ sudo -u hdfs ./bin/hadoop-daemon.sh start datanode + $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs / + $ sudo -u hdfs ./bin/hadoop fs -chmod 755 / + $ sudo -u hdfs ./bin/hadoop fs -mkdir /user + $ sudo -u hdfs ./bin/hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred + $ sudo -u hdfs ./bin/hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred + EOM + + examples_jar = "hadoop-examples-#{version}.jar" + when '2' + log <<-EOM +Note: +You must initialize HDFS in the first installation: + $ cd #{node['grid']['app_root']}/hadoop + $ sudo -u hdfs ./bin/hdfs namenode -format + $ sudo -u hdfs ./sbin/hadoop-daemon.sh start namenode + $ sudo -u hdfs ./sbin/hadoop-daemon.sh start datanode + $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs / + $ sudo -u hdfs ./bin/hadoop fs -chmod 755 / + $ sudo -u hdfs ./bin/hadoop fs -mkdir /user + $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm + $ sudo -u hdfs ./bin/hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm + $ sudo -u hdfs ./bin/hadoop fs -chmod 1777 #{node['grid']['vol_root']}/0/var/log/yarn/nm + $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history + $ sudo -u hdfs ./bin/hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred + $ sudo -u hdfs ./bin/hadoop fs -chmod -R 755 #{node['grid']['vol_root']}/0/var/lib/mapred + EOM + + examples_jar = "share/hadoop/mapreduce/hadoop-mapreduce-examples-#{version}.jar" +end + +if node['hadoop']['with_security'] then + log <<-EOM +Note: +Example MapReduce job execution: + $ sudo -u alice kinit + Password for alice@LOCALDOMAIN: + $ sudo -u alice bin/hadoop jar #{examples_jar} pi \\ + > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10 + EOM +else + log <<-EOM +Note: +Example MapReduce job execution: + $ sudo adduser alice + $ sudo -u hdfs ./bin/hadoop fs -mkdir /user/alice + $ sudo -u hdfs ./bin/hadoop fs -chown alice:alice /user/alice + $ sudo -u alice ./bin/hadoop jar #{examples_jar} pi 5 10 + EOM +end + diff --git a/cookbooks/hadoop/recipes/default.rb b/cookbooks/hadoop/recipes/default.rb index c624dbc..b3a6cc1 100644 --- a/cookbooks/hadoop/recipes/default.rb +++ b/cookbooks/hadoop/recipes/default.rb @@ -17,467 +17,11 @@ # limitations under the License. # -require 'digest/sha2' - -users = nil -limits_files = nil -conf_files = nil - -users_v1 = { - :hadoop => {:name => 'hadoop', :uid => 10001}, - :hdfs => {:name => 'hdfs', :uid => 10002}, - :mapred => {:name => 'mapred', :uid => 10003} -} -users_v2 = users_v1.merge( - {:yarn => {:name => 'yarn', :uid => 10004}} -) - -limits_files_v1 = [ - 'hdfs.conf', - 'mapreduce.conf' -] -limits_files_v2 = limits_files_v1 + ['yarn.conf'] - -conf_files_v1 = [ - 'capacity-scheduler.xml', - 'configuration.xsl', - 'core-site.xml', - 'fair-scheduler.xml', - 'hadoop-env.sh', - 'hadoop-metrics2.properties', - 'hadoop-policy.xml', - 'hdfs-site.xml', - 'hosts.include', - 'hosts.exclude', - 'log4j.properties', - 'mapred-queue-acls.xml', - 'mapred-site.xml', - 'masters', - 'slaves', - #'ssl-client.xml', - #'ssl-server.xml' -] - -conf_files_v2 = \ - conf_files_v1 \ - - [ - 'fair-scheduler.xml', - 'mapred-queue-acls.xml', - 'masters' - ] \ - + [ - 'httpfs-env.sh', - 'httpfs-log4j.properties', - 'httpfs-signature.secret', - 'httpfs-site.xml', - 'mapred-env.sh', - 'yarn-env.sh', - 'yarn-site.xml' - ] - -# for ver. 1.0 -krb5_conf_files = [ - 'krb5-strong.conf', - 'krb5-weak.conf' -] - -def conf_template(conf_dir, middle_version, conf_files, tpl_vars) - source_dir = (middle_version == '2.0') \ - ? "etc-#{middle_version}/hadoop" \ - : "conf-#{middle_version}" - - conf_files.each {|conf_file| - template "#{conf_dir}/#{conf_file}" do - source "#{source_dir}/#{conf_file}" - owner 'root' - group 'root' - mode '0644' - variables(tpl_vars) - end - } -end - -version = node['hadoop']['version'] -major_version = nil -middle_version = nil - -if /^(\d+)\.(\d+)\.(\d+)\.?(\d*)[-\w]*$/ =~ version then - major_version = $1 - middle_version = "#{$1}.#{$2}" -else - Chef::Application.fatal!("Invalid Hadoop version: #{version}") -end - -unless (('1.0.0' <= version && version < '1.2') \ - || middle_version == '2.0') then - Chef::Application.fatal!("Non supported version: #{version}") -end - -hadoop_members = nil -case major_version - when '1' - users = users_v1 - limits_files = limits_files_v1 - conf_files = conf_files_v1 - when '2' - users = users_v2 - limits_files = limits_files_v2 - conf_files = conf_files_v2 -end - -hadoop_members = [] -users.each {|key, user| - if key != :hadoop - hadoop_members.push(user[:name]) - - group user[:name] do - gid user[:uid] - members [] - action :create - not_if "getent group #{user[:name]}" - end - - user user[:name] do - uid user[:uid] - gid user[:uid] - home "/home/#{user[:name]}" - shell '/bin/sh' - password nil - supports :manage_home => false - not_if "getent passwd #{user[:name]}" - end - end -} - -group users[:hadoop][:name] do - gid users[:hadoop][:uid] - members hadoop_members - append true - action :create - not_if "getent group #{users[:hadoop][:name]}" -end - -user users[:hadoop][:name] do - uid users[:hadoop][:uid] - gid users[:hadoop][:uid] - home "/home/#{users[:hadoop][:name]}" - shell '/bin/sh' - password nil - supports :manage_home => false - not_if "getent passwd #{users[:hadoop][:name]}" -end - -active_vol_nums = 0 -node['grid']['max_vol_nums'].to_i.times {|vol_num| - target_vol_dir = "#{node['grid']['vol_root']}/#{vol_num}" - - if vol_num == 0 || FileTest::directory?(target_vol_dir) then - directory "#{target_vol_dir}/var" do - owner 'root' - group 'root' - mode '0755' - action :create - recursive true - end - - %w{lib log}.each {|dir| - directory "#{target_vol_dir}/var/#{dir}" do - owner 'root' - group 'hadoop' - mode '0775' - action :create - recursive true - end - } - - if major_version == '2' then - directory "#{target_vol_dir}/var/log/yarn" do - owner 'yarn' - group 'hadoop' - mode '0755' - action :create - recursive true - end - - directory "#{target_vol_dir}/var/log/yarn/nm" do - owner 'yarn' - group 'hadoop' - mode '0755' - action :create - recursive true - end - - directory "#{target_vol_dir}/var/log/yarn/nm/local" do - owner 'yarn' - group 'hadoop' - mode '0755' - action :create - recursive true - end - end - - directory "#{target_vol_dir}/tmp" do - owner 'root' - group 'root' - mode '1777' - action :create - recursive true - end - - if major_version == '2' then - directory "#{target_vol_dir}/mapred/tmp" do - owner 'root' - group 'root' - mode '1777' - action :create - recursive true - end - end - - if vol_num == 0 then - directory "#{target_vol_dir}/var/run" do - owner 'root' - group 'hadoop' - mode '0775' - action :create - recursive true - end - - directory "#{target_vol_dir}/var/log/hdfs" do - owner 'hdfs' - group 'hdfs' - mode '0755' - action :create - recursive true - end - end +flavor = node['hadoop']['install_flavor'] +case flavor + when 'apache' + include_recipe "hadoop::#{flavor}" else - break - end - active_vol_nums = vol_num + 1 -} -log "This node active volumes: #{active_vol_nums}" - -file_cache_path = Chef::Config[:file_cache_path] -install_root = "#{node['grid']['app_root']}/hadoop-#{version}" -tarball = nil -case major_version - when '1' - tarball = "hadoop-#{version}-bin.tar.gz" - when '2' - tarball = "hadoop-#{version}.tar.gz" -end -tarball_mds = "#{tarball}.mds" -downloaded_tarball = "#{file_cache_path}/#{tarball}" -downloaded_tarball_mds = "#{file_cache_path}/#{tarball_mds}" - -archive_url = node['hadoop']['archive_url'] -if ! FileTest.directory? install_root then - remote_file downloaded_tarball_mds do - source "#{archive_url}/hadoop-#{version}/#{tarball_mds}" - action :create_if_missing - end - - remote_file downloaded_tarball do - source "#{archive_url}/hadoop-#{version}/#{tarball}" - action :create_if_missing - end - - ruby_block "sha256 checksum #{downloaded_tarball}" do - block do -=begin -e.g. md file format -'hadoop-1.1.2-bin.tar.gz: MD5 = 4B 59 F4 81 A7 52 D2 A9 20 3D D7 D0 A9 50 5C - 18 -hadoop-1.1.2-bin.tar.gz: SHA1 = DCCC 01A0 4C42 587D 9DF1 83CA 7DC8 83F7 A6A4 - 8D80 -...' -=end - checksum = File.read(downloaded_tarball_mds). - gsub(/(\s)+/, ''). - scan(/#{tarball}:(.+?)=([0-9A-Z]+)/). - assoc('SHA256')[1] - Chef::Log.info "#{tarball}: SHA256 = #{checksum}" - actual_checksum = Digest::SHA256.file(downloaded_tarball).to_s - Chef::Log.info "#{tarball}: actual SHA256 = #{actual_checksum}" - if ! checksum.casecmp(actual_checksum) then - Chef::Application.fatal!("Invalid SHA256 checksum of #{downloaded_tarball}, expected: #{checksum}") - end - end - action :create - end - - pkg = 'tar' - resources(:package => pkg) rescue package pkg do - action :install - end - - bash "install_hadoop-#{version}" do - code <<-EOC - tar xvzf #{downloaded_tarball} -C #{node['grid']['app_root']} - # for 2.0.x - chown -R root:root #{install_root} - EOC - creates install_root - end -end - -link node['hadoop']['HADOOP_PREFIX'] do - to install_root - action [:delete, :create] -end - -limits_files.each {|limits_file| - template "/etc/security/limits.d/#{limits_file}" do - source "etc/security/limits.d/#{limits_file}" - owner 'root' - group 'root' - mode '0644' - end -} - -conf_dir = nil -case major_version - when '1' - conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/conf" - when '2' - conf_dir = "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop" -end - -tpl_vars = { - :active_vol_nums => active_vol_nums -} -conf_template(conf_dir, middle_version, conf_files, tpl_vars) - -# with security -if node['hadoop']['with_security'] then - directory node['hadoop']['this.keytab.dir'] do - owner 'root' - group 'root' - mode '0755' - action :create - recursive true - end - - jsvc_pkg = nil - case node[:platform_family] - when 'debian' - jsvc_pkg = 'jsvc' - when 'rhel' - jsvc_pkg = 'jakarta-commons-daemon-jsvc' - end - - case major_version - when '1' - if node[:kernel][:machine] != 'x86_64' then - package jsvc_pkg do - action :install - end - - link "#{install_root}/libexec/jsvc.i386" do - to '/usr/bin/jsvc' - end - end - - file "#{node['grid']['app_root']}/hadoop-#{version}/bin/task-controller" do - owner 'root' - group 'mapred' - mode '6050' - end - - template "#{node['grid']['app_root']}/hadoop-#{version}/conf/taskcontroller.cfg" do - source "conf-#{middle_version}/taskcontroller.cfg" - owner 'root' - group 'root' - mode '0400' - variables({ - :active_vol_nums => active_vol_nums - }) - end - when '2' - package jsvc_pkg do - action :install - end - - file "#{node['grid']['app_root']}/hadoop-#{version}/bin/container-executor" do - owner 'root' - group 'yarn' - mode '6050' - end - - template "#{node['grid']['app_root']}/hadoop-#{version}/etc/hadoop/container-executor.cfg" do - source "etc-#{middle_version}/hadoop/container-executor.cfg" - owner 'root' - group 'root' - mode '0400' - variables({ - :active_vol_nums => active_vol_nums - }) - end - end - - if middle_version == '1.0' then - tpl_vars = nil - conf_template(conf_dir, middle_version, krb5_conf_files, tpl_vars) - end -end - -case major_version - when '1' - log <<-EOM -Note: -You must initialize HDFS in the first installation: - $ cd #{node['grid']['app_root']}/hadoop - $ sudo -u hdfs ./bin/hadoop namenode -format - $ sudo -u hdfs ./bin/hadoop-daemon.sh start namenode - $ sudo -u hdfs ./bin/hadoop-daemon.sh start datanode - $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs / - $ sudo -u hdfs ./bin/hadoop fs -chmod 755 / - $ sudo -u hdfs ./bin/hadoop fs -mkdir /user - $ sudo -u hdfs ./bin/hadoop fs -mkdir #{node['grid']['vol_root']}/0/var/lib/mapred - $ sudo -u hdfs ./bin/hadoop fs -chown mapred:mapred #{node['grid']['vol_root']}/0/var/lib/mapred - EOM - - examples_jar = "hadoop-examples-#{version}.jar" - when '2' - log <<-EOM -Note: -You must initialize HDFS in the first installation: - $ cd #{node['grid']['app_root']}/hadoop - $ sudo -u hdfs ./bin/hdfs namenode -format - $ sudo -u hdfs ./sbin/hadoop-daemon.sh start namenode - $ sudo -u hdfs ./sbin/hadoop-daemon.sh start datanode - $ sudo -u hdfs ./bin/hadoop fs -chown hdfs:hdfs / - $ sudo -u hdfs ./bin/hadoop fs -chmod 755 / - $ sudo -u hdfs ./bin/hadoop fs -mkdir /user - $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/log/yarn/nm - $ sudo -u hdfs ./bin/hadoop fs -chown yarn:hadoop #{node['grid']['vol_root']}/0/var/log/yarn/nm - $ sudo -u hdfs ./bin/hadoop fs -chmod 1777 #{node['grid']['vol_root']}/0/var/log/yarn/nm - $ sudo -u hdfs ./bin/hadoop fs -mkdir -p #{node['grid']['vol_root']}/0/var/lib/mapred/history - $ sudo -u hdfs ./bin/hadoop fs -chown -R mapred:hadoop #{node['grid']['vol_root']}/0/var/lib/mapred - $ sudo -u hdfs ./bin/hadoop fs -chmod -R 755 #{node['grid']['vol_root']}/0/var/lib/mapred - EOM - - examples_jar = "share/hadoop/mapreduce/hadoop-mapreduce-examples-#{version}.jar" -end - -if node['hadoop']['with_security'] then - log <<-EOM -Note: -Example MapReduce job execution: - $ sudo -u alice kinit - Password for alice@LOCALDOMAIN: - $ sudo -u alice bin/hadoop jar #{examples_jar} pi \\ - > -D mapreduce.job.acl-view-job=* -D mapreduce.job.acl-modify-job=alice 5 10 - EOM -else - log <<-EOM -Note: -Example MapReduce job execution: - $ sudo adduser alice - $ sudo -u hdfs ./bin/hadoop fs -mkdir /user/alice - $ sudo -u hdfs ./bin/hadoop fs -chown alice:alice /user/alice - $ sudo -u alice ./bin/hadoop jar #{examples_jar} pi 5 10 - EOM + Chef::Application.fatal!("Non supported flavor: #{flavor}") end diff --git a/roles/grid-nm.rb b/roles/grid-nm.rb index bbc4569..f5bb0c8 100644 --- a/roles/grid-nm.rb +++ b/roles/grid-nm.rb @@ -227,7 +227,7 @@ default_attributes( 'authorized_for_all_host_commands' => '*', 'check_ganglia_metric' => { 'enabled' => true, - 'gmetad_host' => "gm90.#{Grid::DOMAIN}" + 'gmetad_host' => "#{this_cluster[:gmetads][:service_hostname]}.#{Grid::DOMAIN}" }, } ) diff --git a/roles/pleiades-nm.rb b/roles/pleiades-nm.rb index bda853d..35437f5 100644 --- a/roles/pleiades-nm.rb +++ b/roles/pleiades-nm.rb @@ -225,8 +225,8 @@ nagios_objects = { { 'host_name' => 'localhost', 'service_description' => 'Cache for check_ganglia_metric', - 'dependent_host_name' => '.*', - #'dependent_hostgroup_name' => 'grid-servers', + #'dependent_host_name' => '.*', # NG! + 'dependent_hostgroup_name' => 'grid-servers', 'dependent_service_description' => '.* via Ganglia$', 'execution_failure_criteria' => 'c,p' } @@ -237,7 +237,9 @@ default_attributes( 'nagios' => { 'objects' => nagios_objects, 'web' => { - 'Allow' => ["from .#{Grid::DOMAIN}"], + 'Allow' => [ + "from .#{Grid::DOMAIN}", + 'from 192.168.0.0/16'], 'AuthType' => 'Kerberos', 'mod_auth_kerb' => { 'KrbAuthRealms' => Grid::REALM, @@ -255,7 +257,7 @@ default_attributes( 'authorized_for_all_host_commands' => '*', 'check_ganglia_metric' => { 'enabled' => true, - 'gmetad_host' => "gm00.#{Grid::DOMAIN}" + 'gmetad_host' => "#{this_cluster[:gmetads][:service_hostname]}.#{Grid::DOMAIN}" }, } )