add ganglia recipe's uni-cast setting support.

diff --git a/cookbooks/ganglia/attributes/default.rb b/cookbooks/ganglia/attributes/default.rb
index ddd9e85..2741bda 100644 (file)
--- a/cookbooks/ganglia/attributes/default.rb
+++ b/cookbooks/ganglia/attributes/default.rb
@@ -23,20 +23,20 @@ default['ganglia']['data_sources'] = [
     :name => 'my cluster',
     # If the polling interval is omitted, 15sec is asssumed.
     :polling_interval => '',
-    # 'address1:port addreses2:port ...'
-    :address_list => 'localhost'
+    # ['address1:port', 'addreses2:port', ...]
+    :address_list => ['localhost']
   }
 ]
 default['ganglia']['gridname'] = 'MyGrid'
-# e.g. '127.0.0.1 169.229.50.165 my.gmetad.org'
-default['ganglia']['trusted_hosts'] = ''
+# e.g. ['127.0.0.1', '169.229.50.165', 'my.gmetad.org']
+default['ganglia']['trusted_hosts'] = []
 # gmond.conf
 default['ganglia']['cluster']['name'] = 'unspecified'
 default['ganglia']['cluster']['owner'] = 'unspecified'
 default['ganglia']['cluster']['latlong'] = 'unspecified'
 default['ganglia']['cluster']['url'] = 'unspecified'
 default['ganglia']['host']['location'] = 'unspecified'
-# multi-cast (default)
+## 1. multi-cast (default)
 default['ganglia']['udp_send_channels'] = [
   {
     'mcast_join' => '239.2.11.71',
@@ -57,6 +57,38 @@ default['ganglia']['tcp_accept_channels'] = [
     'port' => '8649'
   }
 ]
+=begin
+## 2. e.g. uni-cast
+default['ganglia']['udp_send_channels'] = [
+  {
+    'mcast_join' => '',
+    'host' => 'localhost',
+    'port' => '8649',
+    'ttl' => ''
+  }
+]
+default['ganglia']['udp_recv_channels'] = [
+  {
+    'mcast_join' => '',
+    'port' => '8649',
+    'bind' => ''
+  }
+]
+=end
+## 3. automatic unicast setup for huge cluster (e.g. Hadoop DataNodes)
+# if huge_cluster_unicast.enabled, udp_send_channels and udp_recv_channels have no effect.
+default['ganglia']['huge_cluster_unicast'] = {
+  'enabled' => false,
+  'domain' => 'grid.example.com',
+  'node_prefix' => 'dn',
+  'digit_nums' => '5',      # dn#####
+  'padstr' => '0',          # dn00000
+  'first_node_num' => '0',  # 1st node:  dn00000.grid.example.com
+  'last_node_num'  => '999',# last node: dn00999.grid.example.com
+  'head_node_nums' => '2',  # heads of the 1st cluster: dn0000{0..1}.grid.example.com
+  'cluster_size' => '40',   # heads of the 2nd cluster: dn0004{0..1}.grid.example.com
+  'port' => '8649'
+}
 # ganglia-web
 ## default: white list
 default['ganglia']['web']['Order'] = 'Deny,Allow'
@@ -77,21 +109,3 @@ default['ganglia']['web']['mod_auth_kerb']['KrbMethodNegotiate'] = 'on'
 default['ganglia']['web']['mod_auth_kerb']['KrbMethodK5Passwd'] = 'off'
 #default['ganglia']['']
 
-=begin
-# e.g. uni-cast
-default['ganglia']['udp_send_channels'] = [
-  {
-    'mcast_join' => '',
-    'host' => 'localhost',
-    'port' => '8649',
-    'ttl' => '1'
-  }
-]
-default['ganglia']['udp_recv_channels'] = [
-  {
-    'mcast_join' => '',
-    'port' => '8649',
-    'bind' => ''
-  }
-]
-=end

diff --git a/cookbooks/ganglia/templates/centos/etc/ganglia/gmond.conf b/cookbooks/ganglia/templates/centos/etc/ganglia/gmond.conf
index 898329a..0b8430e 100644 (file)
--- a/cookbooks/ganglia/templates/centos/etc/ganglia/gmond.conf
+++ b/cookbooks/ganglia/templates/centos/etc/ganglia/gmond.conf
@@ -45,56 +45,106 @@ host {
 #  ttl = 1
 #}
 <%
-node['ganglia']['udp_send_channels'].each do |channel|
+huge_cluster_unicast = node['ganglia']['huge_cluster_unicast']
+if huge_cluster_unicast['enabled'] then
+  this_node = node[:fqdn]
+  this_domain = huge_cluster_unicast['domain']
+  node_prefix = huge_cluster_unicast['node_prefix']
+  head_node_nums = huge_cluster_unicast['head_node_nums'].to_i
+  digit_nums = huge_cluster_unicast['digit_nums'].to_i
+  padstr = huge_cluster_unicast['padstr']
+  first_node_num = huge_cluster_unicast['first_node_num'].to_i
+  last_node_num = huge_cluster_unicast['last_node_num'].to_i
+  cluster_size = huge_cluster_unicast['cluster_size'].to_i
+
+  head_nodes = []
+  (first_node_num..last_node_num).step(cluster_size) do |i|
+    first_head = node_prefix + i.to_s.rjust(digit_nums, padstr) + ".#{this_domain}"
+    next_first_head = node_prefix + (i + cluster_size).to_s.rjust(digit_nums, padstr) + ".#{this_domain}"
+    if first_head <= this_node \
+      && this_node < next_first_head then
+      (i...(i + head_node_nums)).each do |j|
+        head_nodes.push(node_prefix + j.to_s.rjust(digit_nums, padstr) + ".#{this_domain}")
+      end
+      break
+    end
+  end
+
+  head_nodes.each do |fqdn|
+-%>
+udp_send_channel {
+  host = <%= fqdn %>
+  port = <%= node['ganglia']['huge_cluster_unicast']['port'] %>
+}
+<%
+  end
+else
+  node['ganglia']['udp_send_channels'].each do |channel|
 -%>
 udp_send_channel {
 <%
-  mcast_join = channel['mcast_join']
-  if !mcast_join.nil? && !mcast_join.empty? then
+    mcast_join = channel['mcast_join']
+    if !mcast_join.nil? && !mcast_join.empty? then
 -%>
   mcast_join = <%= mcast_join %>
 <%
-  end
+    end
 -%>
 <%
-  host = channel['host']
-  if !host.nil? && !host.empty? then
+    host = channel['host']
+    if !host.nil? && !host.empty? then
 -%>
   host = <%= host %>
 <%
-  end
+    end
 -%>
   port = <%= channel['port'] %>
+<%
+    ttl = channel['ttl']
+    if !ttl.nil? && !ttl.empty? then
+-%>
   ttl = <%= channel['ttl'] %>
+<%
+    end
+-%>
 }
 <%
+  end
 end
 -%>
 
 /* You can specify as many udp_recv_channels as you like as well. */
 <%
-node['ganglia']['udp_recv_channels'].each do |channel|
+if node['ganglia']['huge_cluster_unicast']['enabled'] then
 -%>
 udp_recv_channel {
+  port = <%= node['ganglia']['huge_cluster_unicast']['port'] %>
+}
 <%
-  mcast_join = channel['mcast_join']
-  if !mcast_join.nil? && !mcast_join.empty? then
+else
+  node['ganglia']['udp_recv_channels'].each do |channel|
+-%>
+udp_recv_channel {
+<%
+    mcast_join = channel['mcast_join']
+    if !mcast_join.nil? && !mcast_join.empty? then
 -%>
   mcast_join = <%= mcast_join %>
 <%
-  end
+    end
 -%>
   port = <%= channel['port'] %>
 <%
-  bind = channel['bind']
-  if !bind.nil? && !bind.empty? then
+    bind = channel['bind']
+    if !bind.nil? && !bind.empty? then
 -%>
   bind = <%= bind %>
 <%
-  end
+    end
 -%>
 }
 <%
+  end
 end
 -%>
 

diff --git a/cookbooks/ganglia/templates/default/etc/ganglia/gmond.conf b/cookbooks/ganglia/templates/default/etc/ganglia/gmond.conf
index f2ee0dd..e3b9d5a 100644 (file)
--- a/cookbooks/ganglia/templates/default/etc/ganglia/gmond.conf
+++ b/cookbooks/ganglia/templates/default/etc/ganglia/gmond.conf
@@ -32,57 +32,107 @@ host {
 /* Feel free to specify as many udp_send_channels as you like.  Gmond 
    used to only support having a single channel */
 <%
-node['ganglia']['udp_send_channels'].each do |channel|
+huge_cluster_unicast = node['ganglia']['huge_cluster_unicast']
+if huge_cluster_unicast['enabled'] then
+  this_node = node[:fqdn]
+  this_domain = huge_cluster_unicast['domain']
+  node_prefix = huge_cluster_unicast['node_prefix']
+  head_node_nums = huge_cluster_unicast['head_node_nums'].to_i
+  digit_nums = huge_cluster_unicast['digit_nums'].to_i
+  padstr = huge_cluster_unicast['padstr']
+  first_node_num = huge_cluster_unicast['first_node_num'].to_i
+  last_node_num = huge_cluster_unicast['last_node_num'].to_i
+  cluster_size = huge_cluster_unicast['cluster_size'].to_i
+
+  head_nodes = []
+  (first_node_num..last_node_num).step(cluster_size) do |i|
+    first_head = node_prefix + i.to_s.rjust(digit_nums, padstr) + ".#{this_domain}"
+    next_first_head = node_prefix + (i + cluster_size).to_s.rjust(digit_nums, padstr) + ".#{this_domain}"
+    if first_head <= this_node \
+      && this_node < next_first_head then
+      (i...(i + head_node_nums)).each do |j|
+        head_nodes.push(node_prefix + j.to_s.rjust(digit_nums, padstr) + ".#{this_domain}")
+      end
+      break
+    end
+  end
+
+  head_nodes.each do |fqdn|
+-%>
+udp_send_channel {
+  host = <%= fqdn %>
+  port = <%= node['ganglia']['huge_cluster_unicast']['port'] %>
+}
+<%
+  end
+else
+  node['ganglia']['udp_send_channels'].each do |channel|
 -%> 
 udp_send_channel {
 <%
-  mcast_join = channel['mcast_join']
-  if !mcast_join.nil? && !mcast_join.empty? then
+    mcast_join = channel['mcast_join']
+    if !mcast_join.nil? && !mcast_join.empty? then
 -%>
   mcast_join = <%= mcast_join %>
 <%
   end
 -%>
 <% 
-  host = channel['host']
-  if !host.nil? && !host.empty? then
+    host = channel['host']
+    if !host.nil? && !host.empty? then
 -%>
   host = <%= host %>
 <%
-  end
+    end
 -%>
   port = <%= channel['port'] %> 
+<%
+    ttl = channel['ttl']
+    if !ttl.nil? && !ttl.empty? then
+-%>
   ttl = <%= channel['ttl'] %> 
+<%
+    end
+-%>
 }
 <%
+  end
 end
 -%>
 
 /* You can specify as many udp_recv_channels as you like as well. */ 
 <%
-node['ganglia']['udp_recv_channels'].each do |channel|
+if node['ganglia']['huge_cluster_unicast']['enabled'] then
+-%>
+udp_recv_channel {
+  port = <%= node['ganglia']['huge_cluster_unicast']['port'] %>
+} 
+<%
+else
+  node['ganglia']['udp_recv_channels'].each do |channel|
 -%> 
 udp_recv_channel { 
 <% 
-  mcast_join = channel['mcast_join']
-  if !mcast_join.nil? && !mcast_join.empty? then
+    mcast_join = channel['mcast_join']
+    if !mcast_join.nil? && !mcast_join.empty? then
 -%>
   mcast_join = <%= mcast_join %>
 <%
-  end
+    end
 -%>
   port = <%= channel['port'] %> 
 <% 
-  bind = channel['bind']
-  if !bind.nil? && !bind.empty? then
+    bind = channel['bind']
+    if !bind.nil? && !bind.empty? then
 -%>
   bind = <%= bind %>
 <%
-  end
+    end
 -%>
-} 
+}
 <%
-end
+  end
+end 
 -%>
 
 /* You can specify as many tcp_accept_channels as you like to share 

diff --git a/cookbooks/hadoop/recipes/default.rb b/cookbooks/hadoop/recipes/default.rb
index a7f392d..3e6b75b 100644 (file)
--- a/cookbooks/hadoop/recipes/default.rb
+++ b/cookbooks/hadoop/recipes/default.rb
@@ -221,11 +221,12 @@ hadoop-1.1.2-bin.tar.gz:   SHA1 = DCCC 01A0 4C42 587D 9DF1  83CA 7DC8 83F7 A6A4
     action :create
   end
 
-  package 'tar' do
+  pkg = 'tar'
+  resources(:package => pkg) rescue package pkg do
     action :install
   end
 
-  bash "install hadoop: #{install_root}" do
+  bash "install_hadoop-#{version}" do
     code <<-EOC
       tar xvzf #{downloaded_tarball} -C #{node['grid']['app_root']}
     EOC

diff --git a/roles/ganglia-gmetad-localhost.rb b/roles/ganglia-gmetad-localhost.rb
index 9da747f..4b56edf 100644 (file)
--- a/roles/ganglia-gmetad-localhost.rb
+++ b/roles/ganglia-gmetad-localhost.rb
@@ -16,7 +16,7 @@ override_attributes(
       {
         :name => 'local',
         :polling_interval => '',
-        :address_list => 'localhost'
+        :address_list => ['localhost']
       }
     ],
     'gridname' => 'Test'

diff --git a/roles/ganglia-gmond-ucast-localhost.rb b/roles/ganglia-gmond-ucast-localhost.rb
index 679a158..1448276 100644 (file)
--- a/roles/ganglia-gmond-ucast-localhost.rb
+++ b/roles/ganglia-gmond-ucast-localhost.rb
@@ -21,7 +21,7 @@ override_attributes(
         'mcast_join' => '',
         'host' => 'localhost',
         'port' => '8649',
-        'ttl' => '1'
+        'ttl' => ''
       }
     ],
     'udp_recv_channels' => [

diff --git a/roles/pleiades-dn00.rb b/roles/pleiades-dn.rb
similarity index 51%
rename from roles/pleiades-dn00.rb
rename to roles/pleiades-dn.rb
index 6df41ec..b45bd1d 100644 (file)
--- a/roles/pleiades-dn00.rb
+++ b/roles/pleiades-dn.rb
@@ -1,5 +1,5 @@
-name 'pleiades-dn00'
-description 'Pleiades DataNode00 node'
+name 'pleiades-dn'
+description 'Pleiades DataNode node'
 
 run_list(
   'role[grid-realm]',
@@ -22,6 +22,7 @@ override_attributes(
       'name' => 'DataNode00',
       'owner' => 'grid.example.com'
     },
+    # multi-cast
     'udp_send_channels' => [
       {
         'mcast_join' => mcast_addr,
@@ -37,5 +38,20 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## uni-cast
+    'huge_cluster_unicast' = {
+      'enabled' => false,
+      'domain' => 'grid.example.com',
+      'node_prefix' => 'dn',
+      'digit_nums' => '5',      # dn#####
+      'padstr' => '0',          # dn00000
+      'first_node_num' => '0',  # 1st node:  dn00000.grid.example.com
+      'last_node_num'  => '999',# last node: dn00999.grid.example.com
+      'head_node_nums' => '2',  # heads of the 1st cluster: dn0000{0..1}.grid.example.com
+      'cluster_size' => '40',   # heads of the 2nd cluster: dn0004{0..1}.grid.example.com
+      'port' => '8649'
+    }
+=end
   }
 )

diff --git a/roles/pleiades-gm.rb b/roles/pleiades-gm.rb
index 513bd2e..6f08bc3 100644 (file)
--- a/roles/pleiades-gm.rb
+++ b/roles/pleiades-gm.rb
@@ -15,6 +15,23 @@ this_domain = 'grid.example.com'
 mcast_addr = '239.2.11.3'
 port = '8649'
 polling_interval = '30'
+# for DataNode
+head_node_nums = 2
+digit_nums = 5
+padstr = '0'
+dn_data_sources = []
+(0...2).step(40) {|v|
+#(0...80).step(40) {|v|
+  first_head = v.to_s.rjust(digit_nums, padstr)
+  last_head = (v + head_node_nums -1).to_s.rjust(digit_nums, padstr)
+  dn_data_sources.push( 
+    {
+      :name => "DataNode#{first_head}",
+      :polling_interval => polling_interval,
+      :address_list => "dn{#{first_head}..#{last_head}}.#{this_domain}".expand
+    }
+  )
+}
 
 #env_run_lists "prod" => ["recipe[apache2]"], "staging" => ["recipe[apache2::staging]"], "_default" => []
 
@@ -40,6 +57,7 @@ override_attributes(
       'name' => 'Ganglia',
       'owner' => this_domain
     },
+    ## multicast
     'udp_send_channels' => [
       {
         'mcast_join' => mcast_addr,
@@ -55,6 +73,30 @@ override_attributes(
         'bind' => mcast_addr
       }
     ],
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "gm00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "gm01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ],
+=end
     # gmetad
     'gridname' => 'Pleiades',
     'trusted_hosts' => ['127.0.0.1'] + "nm0{0..1}.#{this_domain}".expand,
@@ -104,11 +146,6 @@ override_attributes(
         :polling_interval => polling_interval,
         :address_list => "rm0{0..1}.#{this_domain}".expand
       },
-      {
-        :name => 'DataNode00',
-        :polling_interval => polling_interval,
-        :address_list => "dn0000{0..1}.#{this_domain}".expand
-      }
-    ]
+    ] + dn_data_sources
   }
 )

diff --git a/roles/pleiades-gw.rb b/roles/pleiades-gw.rb
index 0ebceda..bbe63c6 100644 (file)
--- a/roles/pleiades-gw.rb
+++ b/roles/pleiades-gw.rb
@@ -38,6 +38,30 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "gw00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "gw01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   },
   'pig' => {
     'version' => '0.11.1'

diff --git a/roles/pleiades-jt.rb b/roles/pleiades-jt.rb
index 4c026d5..37d2bfc 100644 (file)
--- a/roles/pleiades-jt.rb
+++ b/roles/pleiades-jt.rb
@@ -37,5 +37,29 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "jt00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "jt01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   }
 )

diff --git a/roles/pleiades-nm.rb b/roles/pleiades-nm.rb
index 60fff25..c9a7a27 100644 (file)
--- a/roles/pleiades-nm.rb
+++ b/roles/pleiades-nm.rb
@@ -36,5 +36,29 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "nm00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "nm01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   }
 )

diff --git a/roles/pleiades-nn.rb b/roles/pleiades-nn.rb
index 86f4151..d211027 100644 (file)
--- a/roles/pleiades-nn.rb
+++ b/roles/pleiades-nn.rb
@@ -37,5 +37,29 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "nn00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "nn01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   }
 )

diff --git a/roles/pleiades-pxy.rb b/roles/pleiades-pxy.rb
index 6d6cd2c..dcf73d7 100644 (file)
--- a/roles/pleiades-pxy.rb
+++ b/roles/pleiades-pxy.rb
@@ -36,5 +36,29 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "pxy00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "pxy01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   }
 )

diff --git a/roles/pleiades-wf.rb b/roles/pleiades-wf.rb
index 1093b9c..9896d9a 100644 (file)
--- a/roles/pleiades-wf.rb
+++ b/roles/pleiades-wf.rb
@@ -36,5 +36,29 @@ override_attributes(
         'bind' => mcast_addr
       }
     ]
+=begin
+    ## unicast
+    'udp_send_channels' => [
+      {
+        'mcast_join' => '',
+        'host' => "wf00.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      },
+      {
+        'mcast_join' => '',
+        'host' => "wf01.#{this.domain}",
+        'port' => port,
+        'ttl' => ''
+      }
+    ],
+    'udp_recv_channels' => [
+      {
+        'mcast_join' => '',
+        'port' => port,
+        'bind' => ''
+      }
+    ]
+=end
   }
 )

diff --git a/roles/test-on-localhost.rb b/roles/test-on-localhost.rb
index 6aa322a..f6e1176 100644 (file)
--- a/roles/test-on-localhost.rb
+++ b/roles/test-on-localhost.rb
@@ -3,6 +3,7 @@ description 'Testing on local machine'
 
 run_list(
   'role[mocker]',
+  'role[chef_utils]',
   'role[ganglia-gmond-ucast-localhost]',
   'role[ganglia-gmetad-localhost]',
   'role[ganglia-web]',
@@ -37,7 +38,7 @@ default_attributes(
     }
   },
   'ganglia' => {
-    'trusted_hosts' => '127.0.0.1',
+    'trusted_hosts' => ['127.0.0.1'],
     'web' => {
       #'Allow' => ['from .grid.example.com'],
       'AuthType' => 'Kerberos',
@@ -116,4 +117,13 @@ default_attributes(
   }
 )
 
-#override_attributes "apache2" => { "max_children" => "50" }
+override_attributes(
+  'ganglia' => {
+    'trusted_hosts' => ['127.0.0.1'],
+    # for TEST
+    #'huge_cluster_unicast' => {
+    #  'enabled' => true
+    #}
+  }
+)
+