bug fix for hadoop-metrics2 configurations of the Hadoop 2.x series.

diff --git a/.gitignore b/.gitignore
index 5b402f1..fba382d 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,8 @@
 
 .chef/knife.rb
 .berkshelf/config.json
+nodes/*tmp.json
+roles/*tmp.rb
 cookbooks/dsh/files/default/rpms/*.rpm
 
 # Berkshelf

diff --git a/cookbooks/hadoop/recipes/cdh.rb b/cookbooks/hadoop/recipes/cdh.rb
index 01d1b47..f090f6a 100644 (file)
--- a/cookbooks/hadoop/recipes/cdh.rb
+++ b/cookbooks/hadoop/recipes/cdh.rb
@@ -115,7 +115,7 @@ equivalent_apache_hadoop_middle_version \
 equivalent_apache_hadoop_major_version \
   = equivalent_apache_hadoop_middle_version.split('.')[0]
 
-if version >= '5.0.0' then
+if middle_version >= '5.0' then
   conf_files_v2_yarn = conf_files_v2_yarn + [
       'mapred-env.sh'
     ]
@@ -279,8 +279,9 @@ if node['hadoop']['with_security'] then
         })
       end
 
-      # N/A yet!
-      #setup_cgroup(equivalent_apache_hadoop_middle_version)
+      if middle_version >= '5.0' then
+        setup_cgroup(equivalent_apache_hadoop_middle_version)
+      end
     when 'MRv1'
       # hadoop-0.20-mapreduce package includes task-controller
       # (Note: supports both i386 and x86_64.)

diff --git a/cookbooks/hadoop/templates/default/etc-2.0/hadoop/hadoop-metrics2.properties b/cookbooks/hadoop/templates/default/etc-2.0/hadoop/hadoop-metrics2.properties
index 09af4d6..93f88cc 100644 (file)
--- a/cookbooks/hadoop/templates/default/etc-2.0/hadoop/hadoop-metrics2.properties
+++ b/cookbooks/hadoop/templates/default/etc-2.0/hadoop/hadoop-metrics2.properties
@@ -61,12 +61,26 @@
 *.sink.ganglia.dmax=<%= node['hadoop']['metrics2']['*.sink.ganglia.dmax'] %>
 
 <%
-%w{
-  namenode
-  datanode
-  resourcemanager
-  nodemanager
-}.each do |prefix|
+hadoop_daemons = [
+  'namenode',
+  'datanode',
+  'resourcemanager',
+  'nodemanager'
+]
+
+if node['hadoop']['install_flavor'] == 'cdh' \
+  && node['hadoop']['cdh']['resource_negotiator_framework'] == 'MRv1' then
+  hadoop_daemons = [
+    'namenode',
+    'datanode',
+    'jobtracker',
+    'tasktracker',
+    'maptask',
+    'reducetask'
+  ]
+end
+
+hadoop_daemons.each do |prefix|
   servers = node['hadoop']['metrics2']["#{prefix}.sink.ganglia.servers"]
   if !servers.nil? && !servers.empty? then
 -%>

diff --git a/cookbooks/hadoop/templates/default/etc-2.1/hadoop/hadoop-metrics2.properties b/cookbooks/hadoop/templates/default/etc-2.1/hadoop/hadoop-metrics2.properties
index 09af4d6..93f88cc 100644 (file)
--- a/cookbooks/hadoop/templates/default/etc-2.1/hadoop/hadoop-metrics2.properties
+++ b/cookbooks/hadoop/templates/default/etc-2.1/hadoop/hadoop-metrics2.properties
@@ -61,12 +61,26 @@
 *.sink.ganglia.dmax=<%= node['hadoop']['metrics2']['*.sink.ganglia.dmax'] %>
 
 <%
-%w{
-  namenode
-  datanode
-  resourcemanager
-  nodemanager
-}.each do |prefix|
+hadoop_daemons = [
+  'namenode',
+  'datanode',
+  'resourcemanager',
+  'nodemanager'
+]
+
+if node['hadoop']['install_flavor'] == 'cdh' \
+  && node['hadoop']['cdh']['resource_negotiator_framework'] == 'MRv1' then
+  hadoop_daemons = [
+    'namenode',
+    'datanode',
+    'jobtracker',
+    'tasktracker',
+    'maptask',
+    'reducetask'
+  ]
+end
+
+hadoop_daemons.each do |prefix|
   servers = node['hadoop']['metrics2']["#{prefix}.sink.ganglia.servers"]
   if !servers.nil? && !servers.empty? then
 -%>

diff --git a/cookbooks/hadoop/templates/default/etc-2.2/hadoop/hadoop-metrics2.properties b/cookbooks/hadoop/templates/default/etc-2.2/hadoop/hadoop-metrics2.properties
index 09af4d6..93f88cc 100644 (file)
--- a/cookbooks/hadoop/templates/default/etc-2.2/hadoop/hadoop-metrics2.properties
+++ b/cookbooks/hadoop/templates/default/etc-2.2/hadoop/hadoop-metrics2.properties
@@ -61,12 +61,26 @@
 *.sink.ganglia.dmax=<%= node['hadoop']['metrics2']['*.sink.ganglia.dmax'] %>
 
 <%
-%w{
-  namenode
-  datanode
-  resourcemanager
-  nodemanager
-}.each do |prefix|
+hadoop_daemons = [
+  'namenode',
+  'datanode',
+  'resourcemanager',
+  'nodemanager'
+]
+
+if node['hadoop']['install_flavor'] == 'cdh' \
+  && node['hadoop']['cdh']['resource_negotiator_framework'] == 'MRv1' then
+  hadoop_daemons = [
+    'namenode',
+    'datanode',
+    'jobtracker',
+    'tasktracker',
+    'maptask',
+    'reducetask'
+  ]
+end
+
+hadoop_daemons.each do |prefix|
   servers = node['hadoop']['metrics2']["#{prefix}.sink.ganglia.servers"]
   if !servers.nil? && !servers.empty? then
 -%>

diff --git a/nodes/localhost-cdh4-with-ganglia.json b/nodes/localhost-cdh4-with-ganglia.json
new file mode 100644 (file)
index 0000000..db668ac
--- /dev/null
+++ b/nodes/localhost-cdh4-with-ganglia.json
@@ -0,0 +1,5 @@
+{
+  "run_list": [
+    "role[hadoop-pseudo-distributed-cdh4-with-ganglia]"
+  ]
+}

diff --git a/roles/hadoop-pseudo-distributed-cdh4-with-ganglia.rb b/roles/hadoop-pseudo-distributed-cdh4-with-ganglia.rb
new file mode 100644 (file)
index 0000000..5a8c533
--- /dev/null
+++ b/roles/hadoop-pseudo-distributed-cdh4-with-ganglia.rb
@@ -0,0 +1,28 @@
+name 'hadoop-pseudo-distributed-cdh4-with-ganglia'
+description 'Hadoop pseudo distributed mode configurations for CDH 4'
+
+run_list(
+  'role[node_commons]',
+  'role[ganglia-all-in-one-localhost]',
+  'role[hadoop-pseudo-distributed-cdh4]'
+)
+
+#env_run_lists()
+
+default_attributes(
+  'hadoop' => {
+    'metrics2' => {
+      'namenode.sink.ganglia.servers' => 'localhost:8649',
+      'datanode.sink.ganglia.servers' => 'localhost:8649',
+      # for MapReduce 1.x only
+      'jobtracker.sink.ganglia.servers' => 'localhost:8649',
+      'tasktracker.sink.ganglia.servers' => 'localhost:8649',
+      'maptask.sink.ganglia.servers' => 'localhost:8649',
+      'reducetask.sink.ganglia.servers' => 'localhost:8649'
+    }
+  }
+)
+
+override_attributes(
+)
+

diff --git a/roles/test-cdh5-on-localhost.rb b/roles/test-cdh5-on-localhost.rb
index f34c415..fd921fc 100644 (file)
--- a/roles/test-cdh5-on-localhost.rb
+++ b/roles/test-cdh5-on-localhost.rb
@@ -53,8 +53,8 @@ default_attributes(
     'container-executor' => {
       'min.user.id' => '500'
     },
-    #'yarn.nodemanager.linux-container-executor.resources-handler.class' \
-    #  => 'org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler',  # N/A on the CDH4
+    'yarn.nodemanager.linux-container-executor.resources-handler.class' \
+      => 'org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler',  # N/A on the CDH4
     # for 1.x only
     'taskcontroller' => {
       'min.user.id' => '500'