refactor templates of the hadoop and hbase cookbooks.

diff --git a/cookbooks/hadoop/attributes/default.rb b/cookbooks/hadoop/attributes/default.rb
index c5c1885..24a4604 100644 (file)
--- a/cookbooks/hadoop/attributes/default.rb
+++ b/cookbooks/hadoop/attributes/default.rb
@@ -25,8 +25,14 @@ default['hadoop']['archive_url'] = 'http://archive.apache.org/dist/hadoop/core'
 default['hadoop']['with_security'] = false
 ## hadoop-env.sh
 default['hadoop']['HADOOP_PREFIX'] = "#{node['grid']['app_root']}/hadoop"
+default['hadoop']['HADOOP_PID_DIR'] = "#{node['grid']['vol_root']}/0/var/run/$USER"
+default['hadoop']['HADOOP_LOG_DIR'] = "#{node['grid']['vol_root']}/0/var/log/$USER"
 default['hadoop']['HADOOP_CLASSPATH'] = ''
 default['hadoop']['HADOOP_USER_CLASSPATH_FIRST'] = false
+### with_security
+default['hadoop']['HADOOP_SECURE_DN_USER'] = 'hdfs'
+default['hadoop']['HADOOP_SECURE_DN_PID_DIR'] = "#{node['grid']['vol_root']}/0/var/run/${HADOOP_SECURE_DN_USER}"
+default['hadoop']['HADOOP_SECURE_DN_LOG_DIR'] = "#{node['grid']['vol_root']}/0/var/log/${HADOOP_SECURE_DN_USER}"
 ## hadoop-policy.xml
 default['hadoop']['security.refresh.policy.protocol.acl'] = 'hdfs,mapred hadoop'
 default['hadoop']['security.admin.operation.protocol.acl'] = 'hdfs,mapred hadoop'
@@ -36,10 +42,27 @@ default['hadoop']['this.domain'] = 'localhost'
 default['hadoop']['this.realm'] = 'LOCALDOMAIN'
 default['hadoop']['this.keytab.dir'] = "#{node['grid']['etc_root']}/keytabs/#{node['hadoop']['this.cluster.name']}"
 default['hadoop']['this.namenode.fqdn'] = 'localhost'
+default['hadoop']['fs.default.name'] = 'hdfs://${this.namenode.fqdn}:9000'
+default['hadoop']['hadoop.tmp.dir'] = '/tmp/hadoop-${user.name}'
 default['hadoop']['fs.checkpoint.dir'] = "#{node['grid']['vol_root']}/0/var/lib/${user.name}/checkpoint"
+### with_security
+default['hadoop']['hadoop.security.authentication'] = 'kerberos'
+default['hadoop']['hadoop.security.authorization'] = 'true'
+default['hadoop']['hadoop.security.auth_to_local'] = '
+            RULE:[2:$1@$0](.*@${this.realm})s/@.*//
+            RULE:[1:$1@$0](.*@${this.realm})s/@.*//
+            RULE:[2:$1@$0](mapred@.*${this.realm})s/.*/mapred/
+            RULE:[2:$1@$0](hdfs@.*${this.realm})s/.*/hdfs/
+            DEFAULT'
 default['hadoop']['hadoop.security.group.mapping'] = 'org.apache.hadoop.security.JniBasedUnixGroupsMapping'
+default['hadoop']['hadoop.security.groups.cache.secs'] = '14400'
+default['hadoop']['hadoop.kerberos.kinit.command'] = '/usr/bin/kinit'
 default['hadoop']['hadoop.http.filter.initializers'] = 'org.apache.hadoop.security.AuthenticationFilterInitializer'
+default['hadoop']['hadoop.http.authentication.signature.secret.file'] = "#{node['grid']['etc_root']}/hadoop-http-auth-signature-secret"
+default['hadoop']['hadoop.http.authentication.cookie.domain'] = '${this.domain}'
+default['hadoop']['hadoop.http.authentication.type'] = 'kerberos'
 default['hadoop']['hadoop.http.authentication.kerberos.principal'] = 'HTTP/localhost@${this.realm}'
+default['hadoop']['hadoop.http.authentication.kerberos.keytab'] = '${this.keytab.dir}/HTTP.keytab'
 default['hadoop']['hadoop.proxyuser'] = {
   'oozie' => {
     'hosts' => 'localhost',
@@ -53,18 +76,61 @@ default['hadoop']['hadoop.proxyuser'] = {
 ## hdfs-site.xml
 default['hadoop']['this.secondary.namenode.fqdn'] = 'localhost'
 default['hadoop']['dfs.name.dir'] = "#{node['grid']['vol_root']}/0/var/lib/${user.name}/name"
-default['hadoop']['dfs.datanode.kerberos.principal'] = 'hdfs/localhost@${this.realm}'
 default['hadoop']['dfs.replication'] = '1'
+default['hadoop']['dfs.hosts'] = "#{node['grid']['app_root']}/hadoop/conf/hosts.include"
+default['hadoop']['dfs.hosts.exclude'] = "#{node['grid']['app_root']}/hadoop/conf/hosts.exclude"
+### with_security
 default['hadoop']['dfs.cluster.administrators'] = ' hdfs'
+default['hadoop']['dfs.http.port'] = '50070'
+default['hadoop']['dfs.http.address'] = '${this.namenode.fqdn}:${dfs.http.port}'
+#### version >= 1.1.x
+default['hadoop']['hadoop.security.use-weak-http-crypto'] = 'false'
+default['hadoop']['dfs.https.port'] = '50470'
+default['hadoop']['dfs.https.address'] = '${this.namenode.fqdn}:${dfs.https.port}'
+default['hadoop']['dfs.namenode.kerberos.principal'] = 'hdfs/_HOST@${this.realm}'
+default['hadoop']['dfs.namenode.kerberos.https.principal'] = 'host/${this.namenode.fqdn}@${this.realm}'
+default['hadoop']['dfs.namenode.keytab.file'] = '${this.keytab.dir}/nn.keytab'
+default['hadoop']['dfs.secondary.http.port'] = '50090'
+default['hadoop']['dfs.secondary.http.address'] = '${this.secondary.namenode.fqdn}:${dfs.secondary.http.port}'
+default['hadoop']['dfs.secondary.https.port'] = '50495'
+default['hadoop']['dfs.secondary.https.address'] = '${this.secondary.namenode.fqdn}:${dfs.secondary.https.port}'
+default['hadoop']['dfs.secondary.namenode.kerberos.principal'] = 'hdfs/${this.secondary.namenode.fqdn}@${this.realm}'
+default['hadoop']['dfs.secondary.namenode.kerberos.https.principal'] = 'host/${this.secondary.namenode.fqdn}@${this.realm}'
+default['hadoop']['dfs.secondary.namenode.keytab.file'] = '${this.keytab.dir}/cn.keytab'
+default['hadoop']['dfs.block.access.token.enable'] = 'true'
+default['hadoop']['dfs.datanode.kerberos.principal'] = 'hdfs/localhost@${this.realm}'
+default['hadoop']['dfs.datanode.keytab.file'] = '${this.keytab.dir}/dn.keytab'
+default['hadoop']['dfs.datanode.address'] = '0.0.0.0:1004'
+default['hadoop']['dfs.datanode.http.address'] = '0.0.0.0:1006'
+default['hadoop']['dfs.namenode.http-address'] = '${this.namenode.fqdn}:${dfs.http.port}'
+default['hadoop']['dfs.namenode.secondary.http-address'] = '${this.secondary.namenode.fqdn}:${dfs.secondary.http.port}'
+default['hadoop']['dfs.web.authentication.kerberos.principal'] = 'HTTP/_HOST@${this.realm}'
+default['hadoop']['dfs.web.authentication.kerberos.keytab'] = '${this.keytab.dir}/HTTP.keytab'
 ## mapred-site.xml
 default['hadoop']['this.jobtracker.fqdn'] = 'localhost'
 default['hadoop']['this.jobhistory.fqdn'] = 'localhost'
-default['hadoop']['mapreduce.tasktracker.kerberos.principal'] = 'mapred/localhost@${this.realm}'
-default['hadoop']['mapreduce.history.server.embedded'] = true
+default['hadoop']['mapred.job.tracker'] = '${this.jobtracker.fqdn}:9001'
+default['hadoop']['mapred.system.dir'] = "#{node['grid']['vol_root']}/0/var/lib/${user.name}/system"
+default['hadoop']['mapreduce.jobtracker.staging.root.dir'] = '/user'
 default['hadoop']['mapred.job.tracker.persist.jobstatus.active'] = 'true'
 default['hadoop']['mapred.job.tracker.persist.jobstatus.hours'] = '3'
+default['hadoop']['mapred.job.tracker.persist.jobstatus.dir'] = "#{node['grid']['vol_root']}/0/var/lib/mapred/jobstatus"
+default['hadoop']['hadoop.job.history.location'] = "file://#{node['grid']['vol_root']}/0/var/lib/mapred/history"
+default['hadoop']['mapred.job.tracker.history.completed.location'] = "hdfs://#{node['grid']['vol_root']}/0/var/lib/mapred/history/done"
+default['hadoop']['mapred.hosts'] = "#{node['grid']['app_root']}/hadoop/conf/hosts.include"
+default['hadoop']['mapred.hosts.exclude'] = "#{node['grid']['app_root']}/hadoop/conf/hosts.exclude"
+### with_security
+default['hadoop']['mapred.acls.enabled'] = 'true'
 default['hadoop']['mapreduce.cluster.administrators'] = ' mapred'
+default['hadoop']['mapreduce.jobtracker.kerberos.principal'] = 'mapred/_HOST@${this.realm}'
+default['hadoop']['mapreduce.jobtracker.keytab.file'] = '${this.keytab.dir}/jt.keytab'
+default['hadoop']['mapreduce.tasktracker.kerberos.principal'] = 'mapred/localhost@${this.realm}'
+default['hadoop']['mapreduce.tasktracker.keytab.file'] = '${this.keytab.dir}/tt.keytab'
 default['hadoop']['mapred.task.tracker.task-controller'] = 'org.apache.hadoop.mapred.LinuxTaskController'
+default['hadoop']['mapreduce.history.server.embedded'] = 'true'
+default['hadoop']['mapreduce.history.server.http.address'] = '${this.jobhistory.fqdn}:19888'
+default['hadoop']['mapreduce.jobhistory.kerberos.principal'] = 'mapred/${this.jobhistory.fqdn}@${this.realm}'
+default['hadoop']['mapreduce.jobhistory.keytab.file'] = '${this.keytab.dir}/jh.keytab'
 ## mapred-queue-acls.xml
 default['hadoop']['mapred.queue.default.acl-submit-job'] = '*'
 default['hadoop']['mapred.queue.default.acl-administer-jobs'] = ' hadoop,gridops'

diff --git a/cookbooks/hadoop/templates/default/conf-1.0/core-site.xml b/cookbooks/hadoop/templates/default/conf-1.0/core-site.xml
index 929dbe6..699dda0 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.0/core-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.0/core-site.xml
@@ -33,11 +33,11 @@
 
     <property>
         <name>fs.default.name</name>
-        <value>hdfs://${this.namenode.fqdn}:9000</value>
+        <value><%= node['hadoop']['fs.default.name'] %></value>
     </property>
     <property>
         <name>hadoop.tmp.dir</name>
-        <value>/tmp/hadoop-${user.name}</value>
+        <value><%= node['hadoop']['hadoop.tmp.dir'] %></value>
     </property>
     <property>
         <name>fs.checkpoint.dir</name>
@@ -48,7 +48,7 @@
 <% if node['hadoop']['with_security'] then -%>
     <property>
         <name>hadoop.security.authentication</name>
-        <value>kerberos</value>
+        <value><%= node['hadoop']['hadoop.security.authentication'] %></value>
         <description>
             Set the authentication for the cluster. Valid values are: simple or
             kerberos.
@@ -56,20 +56,14 @@
     </property>
     <property>
         <name>hadoop.security.authorization</name>
-        <value>true</value>
+        <value><%= node['hadoop']['hadoop.security.authorization'] %></value>
         <description>
             Enable authorization for different protocols.
         </description>
     </property>
     <property>
         <name>hadoop.security.auth_to_local</name>
-        <value>
-            RULE:[2:$1@$0](.*@${this.realm})s/@.*//
-            RULE:[1:$1@$0](.*@${this.realm})s/@.*//
-            RULE:[2:$1@$0](mapred@.*${this.realm})s/.*/mapred/
-            RULE:[2:$1@$0](hdfs@.*${this.realm})s/.*/hdfs/
-            DEFAULT
-        </value>
+        <value><%= node['hadoop']['hadoop.security.auth_to_local'] %></value>
     </property>
     <property>
         <name>hadoop.security.group.mapping</name>
@@ -77,11 +71,11 @@
     </property>
     <property>
         <name>hadoop.security.groups.cache.secs</name>
-        <value>14400</value>
+        <value><%= node['hadoop']['hadoop.security.groups.cache.secs'] %></value>
     </property>
     <property>
         <name>hadoop.kerberos.kinit.command</name>
-        <value>/usr/bin/kinit</value>
+        <value><%= node['hadoop']['hadoop.kerberos.kinit.command'] %></value>
     </property>
 
     <property>
@@ -94,15 +88,15 @@
     </property>
     <property>
         <name>hadoop.http.authentication.signature.secret.file</name>
-        <value><%= node['grid']['etc_root'] %>/hadoop-http-auth-signature-secret</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.signature.secret.file'] %></value>
     </property>
     <property>
         <name>hadoop.http.authentication.cookie.domain</name>
-        <value>${this.domain}</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.cookie.domain'] %></value>
     </property>
     <property>
         <name>hadoop.http.authentication.type</name>
-        <value>kerberos</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.type'] %></value>
         <description>Defines authentication used for the HTTP web-consoles.
             The supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#.
             The dfeault value is simple.</description>
@@ -126,7 +120,7 @@
     </property>
     <property>
         <name>hadoop.http.authentication.kerberos.keytab</name>
-        <value>${this.keytab.dir}/HTTP.keytab</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.kerberos.keytab'] %></value>
     </property>
 
   <%- node['hadoop']['hadoop.proxyuser'].each do |name, values| -%>

diff --git a/cookbooks/hadoop/templates/default/conf-1.0/hadoop-env.sh b/cookbooks/hadoop/templates/default/conf-1.0/hadoop-env.sh
index 430d1db..1cd4a32 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.0/hadoop-env.sh
+++ b/cookbooks/hadoop/templates/default/conf-1.0/hadoop-env.sh
@@ -3,14 +3,14 @@
 
 export JAVA_HOME=<%= node['java']['java_home'] %>
 # The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=<%= node['grid']['vol_root'] %>/0/var/run/$USER
+export HADOOP_PID_DIR=<%= node['hadoop']['HADOOP_PID_DIR'] %>
 # Where log files are stored. $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/$USER 
+export HADOOP_LOG_DIR=<%= node['hadoop']['HADOOP_LOG_DIR'] %> 
 
 <% if node['hadoop']['with_security'] then -%>
-export HADOOP_SECURE_DN_USER=hdfs
-export HADOOP_SECURE_DN_PID_DIR=<%= node['grid']['vol_root'] %>/0/var/run/${HADOOP_SECURE_DN_USER}
-export HADOOP_SECURE_DN_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/${HADOOP_SECURE_DN_USER}
+export HADOOP_SECURE_DN_USER=<%= node['hadoop']['HADOOP_SECURE_DN_USER'] %>
+export HADOOP_SECURE_DN_PID_DIR=<%= node['hadoop']['HADOOP_SECURE_DN_PID_DIR'] %>
+export HADOOP_SECURE_DN_LOG_DIR=<%= node['hadoop']['HADOOP_SECURE_DN_LOG_DIR'] %>
 <% end -%>
 
 # Extra Java CLASSPATH elements.  Optional.

diff --git a/cookbooks/hadoop/templates/default/conf-1.0/hdfs-site.xml b/cookbooks/hadoop/templates/default/conf-1.0/hdfs-site.xml
index c82c15e..097e4a1 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.0/hdfs-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.0/hdfs-site.xml
@@ -35,7 +35,7 @@ dfs_data_dir = ''
 
     <property>
         <name>dfs.hosts</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.include</value>
+        <value><%= node['hadoop']['dfs.hosts'] %></value>
         <description>
           Names a file that contains a list of hosts that are permitted to connect to the namenode.
           The full pathname of the file must be specified. If the value is empty, all hosts are permitted.
@@ -43,7 +43,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.hosts.exclude</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.exclude</value>
+        <value><%= node['hadoop']['dfs.hosts.exclude'] %></value>
         <description>
           Names a file that contains a list of hosts that are not permitted to connect to the namenode.
           The full pathname of the file must be specified. If the value is empty, no hosts are excluded.
@@ -57,35 +57,35 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.http.port</name>
-        <value>50070</value>
+        <value><%= node['hadoop']['dfs.http.port'] %></value>
         <description>The http port where namenode binds</description>
     </property>
     <property>
         <name>dfs.http.address</name>
-        <value>${this.namenode.fqdn}:${dfs.http.port}</value>
+        <value><%= node['hadoop']['dfs.http.address'] %></value>
         <description>The http address where namenode binds</description>
     </property>
     <!-- for KSSL -->
     <property>
         <name>dfs.https.port</name>
-        <value>50470</value>
+        <value><%= node['hadoop']['dfs.https.port'] %></value>
         <description>The https port where namenode binds</description>
     </property>
     <property>
         <name>dfs.https.address</name>
-        <value>${this.namenode.fqdn}:${dfs.https.port}</value>
+        <value><%= node['hadoop']['dfs.https.address'] %></value>
         <description>The https address where namenode binds</description>
     </property>
     <property>
         <name>dfs.namenode.kerberos.principal</name>
-        <value>hdfs/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.namenode.kerberos.principal'] %></value>
         <!-- _HOST is replaced with the fs.default.name's host name -->
         <!-- <value>hdfs/${this.namenode.fqdn}@${this.realm}</value> -->
         <description>Kerberos principal name for the NameNode</description>
     </property>
     <property>
         <name>dfs.namenode.kerberos.https.principal</name>
-        <value>host/${this.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.namenode.kerberos.https.principal'] %></value>
         <!-- <value>host/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             The Kerberos principal for the host that the NameNode runs on.
@@ -93,7 +93,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.namenode.keytab.file</name>
-        <value>${this.keytab.dir}/nn.keytab</value>
+        <value><%= node['hadoop']['dfs.namenode.keytab.file'] %></value>
         <description>
             Combined keytab file containing the namenode service and host
             principals.
@@ -101,28 +101,28 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.http.port</name>
-        <value>50090</value>
+        <value><%= node['hadoop']['dfs.secondary.http.port'] %></value>
         <description>The http port where secondary namenode binds</description>
     </property>
     <property>
         <name>dfs.secondary.http.address</name>
-        <value>${this.secondary.namenode.fqdn}:${dfs.secondary.http.port}</value>
+        <value><%= node['hadoop']['dfs.secondary.http.address'] %></value>
         <description>The http address where secondary namenode binds</description>
     </property>
     <!-- for KSSL -->
     <property>
         <name>dfs.secondary.https.port</name>
-        <value>50495</value>
+        <value><%= node['hadoop']['dfs.secondary.https.port'] %></value>
         <description>The https port where secondary namenode binds</description>
     </property>
     <property>
         <name>dfs.secondary.https.address</name>
-        <value>${this.secondary.namenode.fqdn}:${dfs.secondary.https.port}</value>
+        <value><%= node['hadoop']['dfs.secondary.https.address'] %></value>
         <description>The https address where secondary namenode binds</description>
     </property>
     <property>
         <name>dfs.secondary.namenode.kerberos.principal</name>
-        <value>hdfs/${this.secondary.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.kerberos.principal'] %></value>
         <!-- <value>hdfs/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             Kerberos principal name for the secondary NameNode.
@@ -130,7 +130,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.namenode.kerberos.https.principal</name>
-        <value>host/${this.secondary.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.kerberos.https.principal'] %></value>
         <!-- <value>host/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             The Kerberos principal for the host that the secondary NameNode
@@ -139,7 +139,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.namenode.keytab.file</name>
-        <value>${this.keytab.dir}/cn.keytab</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.keytab.file'] %></value>
         <description>
             Combined keytab file containing the namenode service and host
             principals.
@@ -147,7 +147,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.block.access.token.enable</name>
-        <value>true</value>
+        <value><%= node['hadoop']['dfs.block.access.token.enable'] %></value>
         <description>
             If "true", access tokens are used as capabilities for accessing
             datanodes.
@@ -165,7 +165,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.datanode.keytab.file</name>
-        <value>${this.keytab.dir}/dn.keytab</value>
+        <value><%= node['hadoop']['dfs.datanode.keytab.file'] %></value>
         <description>
             The filename of the keytab file for the DataNode.
         </description>
@@ -173,28 +173,28 @@ dfs_data_dir = ''
 
     <property>
         <name>dfs.datanode.address</name>
-        <value>0.0.0.0:1004</value>
+        <value><%= node['hadoop']['dfs.datanode.address'] %></value>
     </property>
     <property>
         <name>dfs.datanode.http.address</name>
-        <value>0.0.0.0:1006</value>
+        <value><%= node['hadoop']['dfs.datanode.http.address'] %></value>
     </property>
 
     <property>
         <name>dfs.namenode.http-address</name>
-        <value>${this.namenode.fqdn}:50070</value>
+        <value><%= node['hadoop']['dfs.namenode.http-address'] %></value>
     </property>
     <property>
         <name>dfs.namenode.secondary.http-address</name>
-        <value>${this.secondary.namenode.fqdn}:50090</value>
+        <value><%= node['hadoop']['dfs.namenode.secondary.http-address'] %></value>
     </property>
     <property>
         <name>dfs.web.authentication.kerberos.principal</name>
-        <value>HTTP/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.web.authentication.kerberos.principal'] %></value>
     </property>
     <property>
         <name>dfs.web.authentication.kerberos.keytab</name>
-        <value>${this.keytab.dir}/HTTP.keytab</value>
+        <value><%= node['hadoop']['dfs.web.authentication.kerberos.keytab'] %></value>
     </property>
 <% end -%>
 <%

diff --git a/cookbooks/hadoop/templates/default/conf-1.0/mapred-site.xml b/cookbooks/hadoop/templates/default/conf-1.0/mapred-site.xml
index 7b73a1a..19bce46 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.0/mapred-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.0/mapred-site.xml
@@ -18,15 +18,15 @@
 
     <property>
         <name>mapred.job.tracker</name>
-        <value>${this.jobtracker.fqdn}:9001</value>
+        <value><%= node['hadoop']['mapred.job.tracker'] %></value>
     </property>
     <property>
         <name>mapred.system.dir</name>
-        <value><%= node['grid']['vol_root'] %>/0/var/lib/${user.name}/system</value>
+        <value><%= node['hadoop']['mapred.system.dir'] %></value>
     </property>
     <property>
         <name>mapreduce.jobtracker.staging.root.dir</name>
-        <value>/user</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.staging.root.dir'] %></value>
     </property>
 
     <property>
@@ -47,7 +47,7 @@
     </property>
     <property>
         <name>mapred.job.tracker.persist.jobstatus.dir</name>
-        <value><%= node['grid']['vol_root'] %>/0/var/lib/mapred/jobstatus</value>
+        <value><%= node['hadoop']['mapred.job.tracker.persist.jobstatus.dir'] %></value>
         <description>The directory where the job status information is persisted
           in a file system to be available after it drops of the memory queue and
           between jobtracker restarts. (default: /jobtracker/jobsInfo)
@@ -56,12 +56,12 @@
 
     <property>
         <name>hadoop.job.history.location</name>
-        <value>file://<%= node['grid']['vol_root'] %>/0/var/lib/mapred/history</value>
+        <value><%= node['hadoop']['hadoop.job.history.location'] %></value>
         <description>hdfs:// is UNusable.</description>
     </property>
     <property>
         <name>mapred.job.tracker.history.completed.location</name>
-        <value>hdfs://<%= node['grid']['vol_root'] %>/0/var/lib/mapred/history/done</value>
+        <value><%= node['hadoop']['mapred.job.tracker.history.completed.location'] %></value>
     </property>
 
 <%
@@ -99,7 +99,7 @@ mapred_temp_dir = ''
 
     <property>
         <name>mapred.hosts</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.include</value>
+        <value><%= node['hadoop']['mapred.hosts'] %></value>
         <description>
           Names a file that contains the list of nodes that may connect to the jobtracker.
           If the value is empty, all hosts are permitted.
@@ -107,7 +107,7 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapred.hosts.exclude</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.exclude</value>
+        <value><%= node['hadoop']['mapred.hosts.exclude'] %></value>
         <description>
           Names a file that contains the list of hosts that should be excluded by the jobtracker.
           If the value is empty, no hosts are excluded.
@@ -117,7 +117,7 @@ mapred_temp_dir = ''
 <% if node['hadoop']['with_security'] then -%>
     <property>
         <name>mapred.acls.enabled</name>
-        <value>true</value>
+        <value><%= node['hadoop']['mapred.acls.enabled'] %></value>
     </property>
     <property>
         <name>mapreduce.cluster.administrators</name>
@@ -125,13 +125,13 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.jobtracker.kerberos.principal</name>
-        <value>mapred/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.kerberos.principal'] %></value>
         <!-- _HOST is replaced with the mapred.job.tracker's host name -->
         <!-- <value>mapred/${this.jobtracker.fqdn}@${this.realm}</value> -->
     </property>
     <property>
         <name>mapreduce.jobtracker.keytab.file</name>
-        <value>${this.keytab.dir}/jt.keytab</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.keytab.file'] %></value>
     </property>
     <property>
         <name>mapreduce.tasktracker.kerberos.principal</name>
@@ -140,7 +140,7 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.tasktracker.keytab.file</name>
-        <value>${this.keytab.dir}/tt.keytab</value>
+        <value><%= node['hadoop']['mapreduce.tasktracker.keytab.file'] %></value>
     </property>
 
     <property>
@@ -155,15 +155,15 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.history.server.http.address</name>
-        <value>${this.jobhistory.fqdn}:19888</value>
+        <value><%= node['hadoop']['mapreduce.history.server.http.address'] %></value>
     </property>
     <property>
         <name>mapreduce.jobhistory.kerberos.principal</name>
-        <value>mapred/${this.jobhistory.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['mapreduce.jobhistory.kerberos.principal'] %></value>
     </property>
     <property>
         <name>mapreduce.jobhistory.keytab.file</name>
-        <value>${this.keytab.dir}/jh.keytab</value>
+        <value><%= node['hadoop']['mapreduce.jobhistory.keytab.file'] %></value>
     </property>
 <% end -%>
 <%

diff --git a/cookbooks/hadoop/templates/default/conf-1.1/core-site.xml b/cookbooks/hadoop/templates/default/conf-1.1/core-site.xml
index 8372a03..699dda0 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.1/core-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.1/core-site.xml
@@ -14,7 +14,7 @@
         <value><%= node['hadoop']['this.domain'] %></value>
         <!-- <value>grid.example.com</value> -->
     </property>
-<% if node['hadoop']['with_security'] then %>
+<% if node['hadoop']['with_security'] then -%>
     <property>
         <name>this.realm</name>
         <value><%= node['hadoop']['this.realm'] %></value>
@@ -24,7 +24,7 @@
         <name>this.keytab.dir</name>
         <value><%= node['hadoop']['this.keytab.dir'] %></value>
     </property>
-<% end %>
+<% end -%>
     <property>
         <name>this.namenode.fqdn</name>
         <value><%= node['hadoop']['this.namenode.fqdn'] %></value>
@@ -33,11 +33,11 @@
 
     <property>
         <name>fs.default.name</name>
-        <value>hdfs://${this.namenode.fqdn}:9000</value>
+        <value><%= node['hadoop']['fs.default.name'] %></value>
     </property>
     <property>
         <name>hadoop.tmp.dir</name>
-        <value>/tmp/hadoop-${user.name}</value>
+        <value><%= node['hadoop']['hadoop.tmp.dir'] %></value>
     </property>
     <property>
         <name>fs.checkpoint.dir</name>
@@ -45,10 +45,10 @@
         <!-- <value><%= node['grid']['vol_root'] %>/0/var/lib/${user.name}/checkpoint,/export/home/${user.name}/var/lib/checkpoint</value> -->
     </property>
 
-<% if node['hadoop']['with_security'] then %>
+<% if node['hadoop']['with_security'] then -%>
     <property>
         <name>hadoop.security.authentication</name>
-        <value>kerberos</value>
+        <value><%= node['hadoop']['hadoop.security.authentication'] %></value>
         <description>
             Set the authentication for the cluster. Valid values are: simple or
             kerberos.
@@ -56,20 +56,14 @@
     </property>
     <property>
         <name>hadoop.security.authorization</name>
-        <value>true</value>
+        <value><%= node['hadoop']['hadoop.security.authorization'] %></value>
         <description>
             Enable authorization for different protocols.
         </description>
     </property>
     <property>
         <name>hadoop.security.auth_to_local</name>
-        <value>
-            RULE:[2:$1@$0](.*@${this.realm})s/@.*//
-            RULE:[1:$1@$0](.*@${this.realm})s/@.*//
-            RULE:[2:$1@$0](mapred@.*${this.realm})s/.*/mapred/
-            RULE:[2:$1@$0](hdfs@.*${this.realm})s/.*/hdfs/
-            DEFAULT
-        </value>
+        <value><%= node['hadoop']['hadoop.security.auth_to_local'] %></value>
     </property>
     <property>
         <name>hadoop.security.group.mapping</name>
@@ -77,11 +71,11 @@
     </property>
     <property>
         <name>hadoop.security.groups.cache.secs</name>
-        <value>14400</value>
+        <value><%= node['hadoop']['hadoop.security.groups.cache.secs'] %></value>
     </property>
     <property>
         <name>hadoop.kerberos.kinit.command</name>
-        <value>/usr/bin/kinit</value>
+        <value><%= node['hadoop']['hadoop.kerberos.kinit.command'] %></value>
     </property>
 
     <property>
@@ -94,15 +88,15 @@
     </property>
     <property>
         <name>hadoop.http.authentication.signature.secret.file</name>
-        <value><%= node['grid']['etc_root'] %>/hadoop-http-auth-signature-secret</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.signature.secret.file'] %></value>
     </property>
     <property>
         <name>hadoop.http.authentication.cookie.domain</name>
-        <value>${this.domain}</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.cookie.domain'] %></value>
     </property>
     <property>
         <name>hadoop.http.authentication.type</name>
-        <value>kerberos</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.type'] %></value>
         <description>Defines authentication used for the HTTP web-consoles.
             The supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#.
             The dfeault value is simple.</description>
@@ -126,10 +120,10 @@
     </property>
     <property>
         <name>hadoop.http.authentication.kerberos.keytab</name>
-        <value>${this.keytab.dir}/HTTP.keytab</value>
+        <value><%= node['hadoop']['hadoop.http.authentication.kerberos.keytab'] %></value>
     </property>
 
-  <% node['hadoop']['hadoop.proxyuser'].each do |name, values| %>
+  <%- node['hadoop']['hadoop.proxyuser'].each do |name, values| -%>
     <property>
         <name>hadoop.proxyuser.<%= name %>.hosts</name>
         <value><%= values['hosts'] %></value>
@@ -138,15 +132,14 @@
         <name>hadoop.proxyuser.<%= name %>.groups</name>
         <value><%= values['groups'] %></value>
     </property>
-  <% end %>
-<% end %>
-
+  <%- end -%>
+<% end -%>
 <%
 this_file = 'core-site.xml'
 if defined? node['hadoop']['extra_configs'][this_file] \
   && node['hadoop']['extra_configs'][this_file] != nil then
   node['hadoop']['extra_configs'][this_file].each do |key,value|
-%>
+-%>
     <property>
         <name><%= key %></name>
         <value><%= value %></value>
@@ -154,5 +147,5 @@ if defined? node['hadoop']['extra_configs'][this_file] \
 <%
   end
 end 
-%>
+-%>
 </configuration>

diff --git a/cookbooks/hadoop/templates/default/conf-1.1/hadoop-env.sh b/cookbooks/hadoop/templates/default/conf-1.1/hadoop-env.sh
index 97f7ba5..5abfd2f 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.1/hadoop-env.sh
+++ b/cookbooks/hadoop/templates/default/conf-1.1/hadoop-env.sh
@@ -3,15 +3,15 @@
 
 export JAVA_HOME=<%= node['java']['java_home'] %>
 # The directory where pid files are stored. /tmp by default.
-export HADOOP_PID_DIR=<%= node['grid']['vol_root'] %>/0/var/run/$USER
+export HADOOP_PID_DIR=<%= node['hadoop']['HADOOP_PID_DIR'] %>
 # Where log files are stored. $HADOOP_HOME/logs by default.
-export HADOOP_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/$USER 
+export HADOOP_LOG_DIR=<%= node['hadoop']['HADOOP_LOG_DIR'] %> 
 
-<% if node['hadoop']['with_security'] then %>
-export HADOOP_SECURE_DN_USER=hdfs
-export HADOOP_SECURE_DN_PID_DIR=<%= node['grid']['vol_root'] %>/0/var/run/${HADOOP_SECURE_DN_USER}
-export HADOOP_SECURE_DN_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/${HADOOP_SECURE_DN_USER}
-<% end %>
+<% if node['hadoop']['with_security'] then -%>
+export HADOOP_SECURE_DN_USER=<%= node['hadoop']['HADOOP_SECURE_DN_USER'] %>
+export HADOOP_SECURE_DN_PID_DIR=<%= node['hadoop']['HADOOP_SECURE_DN_PID_DIR'] %>
+export HADOOP_SECURE_DN_LOG_DIR=<%= node['hadoop']['HADOOP_SECURE_DN_LOG_DIR'] %>
+<% end -%>
 
 # Extra Java CLASSPATH elements.  Optional.
 if [ x"$HADOOP_CLASSPATH" = x ]; then
@@ -23,7 +23,7 @@ fi
 export HADOOP_USER_CLASSPATH_FIRST=<%= node['hadoop']['HADOOP_USER_CLASSPATH_FIRST'] %>
 
 <% if node['hadoop']['this.cluster.name'] != 'localhost' \
-   && node['hadoop']['with_security'] then %>
+   && node['hadoop']['with_security'] then -%>
 FQDN=`hostname --fqdn`
 CLUSTER_NAME=<%= node['hadoop']['this.cluster.name'] %>
 DOMAIN=<%= node['hadoop']['this.domain'] %>
@@ -32,7 +32,7 @@ export HADOOP_SECONDARYNAMENODE_OPTS="-Dthis.fqdn=${CLUSTER_NAME}-cn.${DOMAIN} $
 export HADOOP_DATANODE_OPTS="-Dthis.fqdn=${FQDN} ${HADOOP_DATANODE_OPTS}"
 export HADOOP_JOBTRACKER_OPTS="-Dthis.fqdn=${CLUSTER_NAME}-jt.${DOMAIN} ${HADOOP_JOBTRACKER_OPTS}"
 export HADOOP_TASKTRACKER_OPTS="-Dthis.fqdn=${FQDN} ${HADOOP_TASKTRACKER_OPTS}"
-<% end %>
+<% end -%>
 
 
 # The only required environment variable is JAVA_HOME.  All others are
@@ -94,10 +94,10 @@ this_file = 'hadoop-env.sh'
 if defined? node['hadoop']['extra_configs'][this_file] \
   && node['hadoop']['extra_configs'][this_file] != nil then
   node['hadoop']['extra_configs'][this_file].each do |key, value|
-%>
+-%>
 export <%= key %>=<%= value %>
 <%
   end
 end
-%>
+-%>
 

diff --git a/cookbooks/hadoop/templates/default/conf-1.1/hdfs-site.xml b/cookbooks/hadoop/templates/default/conf-1.1/hdfs-site.xml
index 220cacc..4020992 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.1/hdfs-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.1/hdfs-site.xml
@@ -21,7 +21,7 @@ dfs_data_dir = ''
   dfs_data_dir = dfs_data_dir == '' ? '' : "#{dfs_data_dir},"
   dfs_data_dir = "#{dfs_data_dir}#{node['grid']['vol_root']}/#{vol_num}/var/lib/${user.name}/data"
 }
-%>
+-%>
     <property>
         <name>dfs.data.dir</name>
         <value><%= dfs_data_dir %></value>
@@ -35,7 +35,7 @@ dfs_data_dir = ''
 
     <property>
         <name>dfs.hosts</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.include</value>
+        <value><%= node['hadoop']['dfs.hosts'] %></value>
         <description>
           Names a file that contains a list of hosts that are permitted to connect to the namenode.
           The full pathname of the file must be specified. If the value is empty, all hosts are permitted.
@@ -43,54 +43,55 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.hosts.exclude</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.exclude</value>
+        <value><%= node['hadoop']['dfs.hosts.exclude'] %></value>
         <description>
           Names a file that contains a list of hosts that are not permitted to connect to the namenode.
           The full pathname of the file must be specified. If the value is empty, no hosts are excluded.
         </description>
     </property>
 
-<% if node['hadoop']['with_security'] then %>
+<% if node['hadoop']['with_security'] then -%>
     <property>
         <name>dfs.cluster.administrators</name>
         <value><%= node['hadoop']['dfs.cluster.administrators'] %></value>
     </property>
     <property>
         <name>dfs.http.port</name>
-        <value>50070</value>
+        <value><%= node['hadoop']['dfs.http.port'] %></value>
         <description>The http port where namenode binds</description>
     </property>
     <property>
         <name>dfs.http.address</name>
-        <value>${this.namenode.fqdn}:${dfs.http.port}</value>
+        <value><%= node['hadoop']['dfs.http.address'] %></value>
         <description>The http address where namenode binds</description>
     </property>
-    <!-- for KSSL (NOT RECOMMENDED)
+    <!-- for KSSL (NOT RECOMMENDED) -->
     <property>
         <name>hadoop.security.use-weak-http-crypto</name>
-        <value>true</value>
+        <value><%= node['hadoop']['hadoop.security.use-weak-http-crypto'] %></value>
     </property>
+  <% if node['hadoop']['hadoop.security.use-weak-http-crypto'] == 'true' then -%>
     <property>
         <name>dfs.https.port</name>
-        <value>50470</value>
+        <value><%= node['hadoop']['dfs.https.port'] %></value>
         <description>The https port where namenode binds</description>
     </property>
     <property>
         <name>dfs.https.address</name>
-        <value>${this.namenode.fqdn}:${dfs.https.port}</value>
+        <value><%= node['hadoop']['dfs.https.address'] %></value>
         <description>The https address where namenode binds</description>
     </property>
-    -->
+  <% end -%>
     <property>
         <name>dfs.namenode.kerberos.principal</name>
-        <value>hdfs/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.namenode.kerberos.principal'] %></value>
         <!-- _HOST is replaced with the fs.default.name's host name -->
         <!-- <value>hdfs/${this.namenode.fqdn}@${this.realm}</value> -->
         <description>Kerberos principal name for the NameNode</description>
     </property>
     <property>
         <name>dfs.namenode.kerberos.https.principal</name>
-        <value>host/${this.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.namenode.kerberos.https.principal'] %></value>
         <!-- <value>host/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             The Kerberos principal for the host that the NameNode runs on.
@@ -98,7 +99,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.namenode.keytab.file</name>
-        <value>${this.keytab.dir}/nn.keytab</value>
+        <value><%= node['hadoop']['dfs.namenode.keytab.file'] %></value>
         <description>
             Combined keytab file containing the namenode service and host
             principals.
@@ -106,29 +107,30 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.http.port</name>
-        <value>50090</value>
+        <value><%= node['hadoop']['dfs.secondary.http.port'] %></value>
         <description>The http port where secondary namenode binds</description>
     </property>
     <property>
         <name>dfs.secondary.http.address</name>
-        <value>${this.secondary.namenode.fqdn}:${dfs.secondary.http.port}</value>
+        <value><%= node['hadoop']['dfs.secondary.http.address'] %></value>
         <description>The http address where secondary namenode binds</description>
     </property>
-    <!-- for KSSL (NOT RECOMMENDED)
+  <% if node['hadoop']['hadoop.security.use-weak-http-crypto'] == 'true' then -%>
+    <!-- for KSSL (NOT RECOMMENDED) -->
     <property>
         <name>dfs.secondary.https.port</name>
-        <value>50495</value>
+        <value><%= node['hadoop']['dfs.secondary.https.port'] %></value>
         <description>The https port where secondary namenode binds</description>
     </property>
     <property>
         <name>dfs.secondary.https.address</name>
-        <value>${this.secondary.namenode.fqdn}:${dfs.secondary.https.port}</value>
+        <value><%= node['hadoop']['dfs.secondary.https.address'] %></value>
         <description>The https address where secondary namenode binds</description>
     </property>
-    -->
+  <% end -%>
     <property>
         <name>dfs.secondary.namenode.kerberos.principal</name>
-        <value>hdfs/${this.secondary.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.kerberos.principal'] %></value>
         <!-- <value>hdfs/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             Kerberos principal name for the secondary NameNode.
@@ -136,7 +138,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.namenode.kerberos.https.principal</name>
-        <value>host/${this.secondary.namenode.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.kerberos.https.principal'] %></value>
         <!-- <value>host/_HOST@${this.realm}</value> v1.0.4: NG! -->
         <description>
             The Kerberos principal for the host that the secondary NameNode
@@ -145,7 +147,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.secondary.namenode.keytab.file</name>
-        <value>${this.keytab.dir}/cn.keytab</value>
+        <value><%= node['hadoop']['dfs.secondary.namenode.keytab.file'] %></value>
         <description>
             Combined keytab file containing the namenode service and host
             principals.
@@ -153,7 +155,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.block.access.token.enable</name>
-        <value>true</value>
+        <value><%= node['hadoop']['dfs.block.access.token.enable'] %></value>
         <description>
             If "true", access tokens are used as capabilities for accessing
             datanodes.
@@ -171,7 +173,7 @@ dfs_data_dir = ''
     </property>
     <property>
         <name>dfs.datanode.keytab.file</name>
-        <value>${this.keytab.dir}/dn.keytab</value>
+        <value><%= node['hadoop']['dfs.datanode.keytab.file'] %></value>
         <description>
             The filename of the keytab file for the DataNode.
         </description>
@@ -179,37 +181,36 @@ dfs_data_dir = ''
 
     <property>
         <name>dfs.datanode.address</name>
-        <value>0.0.0.0:1004</value>
+        <value><%= node['hadoop']['dfs.datanode.address'] %></value>
     </property>
     <property>
         <name>dfs.datanode.http.address</name>
-        <value>0.0.0.0:1006</value>
+        <value><%= node['hadoop']['dfs.datanode.http.address'] %></value>
     </property>
 
     <property>
         <name>dfs.namenode.http-address</name>
-        <value>${this.namenode.fqdn}:50070</value>
+        <value><%= node['hadoop']['dfs.namenode.http-address'] %></value>
     </property>
     <property>
         <name>dfs.namenode.secondary.http-address</name>
-        <value>${this.secondary.namenode.fqdn}:50090</value>
+        <value><%= node['hadoop']['dfs.namenode.secondary.http-address'] %></value>
     </property>
     <property>
         <name>dfs.web.authentication.kerberos.principal</name>
-        <value>HTTP/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['dfs.web.authentication.kerberos.principal'] %></value>
     </property>
     <property>
         <name>dfs.web.authentication.kerberos.keytab</name>
-        <value>${this.keytab.dir}/HTTP.keytab</value>
+        <value><%= node['hadoop']['dfs.web.authentication.kerberos.keytab'] %></value>
     </property>
-<% end %>
-
+<% end -%>
 <%
 this_file = 'hdfs-site.xml'
 if defined? node['hadoop']['extra_configs'][this_file] \
   && node['hadoop']['extra_configs'][this_file] != nil then
   node['hadoop']['extra_configs'][this_file].each do |key,value|
-%>
+-%>
     <property>
         <name><%= key %></name>
         <value><%= value %></value>
@@ -217,5 +218,5 @@ if defined? node['hadoop']['extra_configs'][this_file] \
 <%
   end
 end 
-%>
+-%>
 </configuration>

diff --git a/cookbooks/hadoop/templates/default/conf-1.1/mapred-site.xml b/cookbooks/hadoop/templates/default/conf-1.1/mapred-site.xml
index d17aa8a..19bce46 100644 (file)
--- a/cookbooks/hadoop/templates/default/conf-1.1/mapred-site.xml
+++ b/cookbooks/hadoop/templates/default/conf-1.1/mapred-site.xml
@@ -18,15 +18,15 @@
 
     <property>
         <name>mapred.job.tracker</name>
-        <value>${this.jobtracker.fqdn}:9001</value>
+        <value><%= node['hadoop']['mapred.job.tracker'] %></value>
     </property>
     <property>
         <name>mapred.system.dir</name>
-        <value><%= node['grid']['vol_root'] %>/0/var/lib/${user.name}/system</value>
+        <value><%= node['hadoop']['mapred.system.dir'] %></value>
     </property>
     <property>
         <name>mapreduce.jobtracker.staging.root.dir</name>
-        <value>/user</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.staging.root.dir'] %></value>
     </property>
 
     <property>
@@ -47,7 +47,7 @@
     </property>
     <property>
         <name>mapred.job.tracker.persist.jobstatus.dir</name>
-        <value><%= node['grid']['vol_root'] %>/0/var/lib/mapred/jobstatus</value>
+        <value><%= node['hadoop']['mapred.job.tracker.persist.jobstatus.dir'] %></value>
         <description>The directory where the job status information is persisted
           in a file system to be available after it drops of the memory queue and
           between jobtracker restarts. (default: /jobtracker/jobsInfo)
@@ -56,12 +56,12 @@
 
     <property>
         <name>hadoop.job.history.location</name>
-        <value>file://<%= node['grid']['vol_root'] %>/0/var/lib/mapred/history</value>
+        <value><%= node['hadoop']['hadoop.job.history.location'] %></value>
         <description>hdfs:// is UNusable.</description>
     </property>
     <property>
         <name>mapred.job.tracker.history.completed.location</name>
-        <value>hdfs://<%= node['grid']['vol_root'] %>/0/var/lib/mapred/history/done</value>
+        <value><%= node['hadoop']['mapred.job.tracker.history.completed.location'] %></value>
     </property>
 
 <%
@@ -70,7 +70,7 @@ mapred_local_dir = ''
   mapred_local_dir = mapred_local_dir == '' ? '' : "#{mapred_local_dir},"
   mapred_local_dir = "#{mapred_local_dir}#{node['grid']['vol_root']}/#{vol_num}/var/lib/mapred/local"
 }
-%>
+-%>
     <property>
         <name>mapred.local.dir</name>
         <value><%= mapred_local_dir %></value>
@@ -87,7 +87,7 @@ mapred_temp_dir = ''
   mapred_temp_dir = mapred_temp_dir == '' ? '' : "#{mapred_temp_dir},"
   mapred_temp_dir = "#{mapred_temp_dir}#{node['grid']['vol_root']}/#{vol_num}/tmp/${user.name}/mapred"
 }
-%>
+-%>
     <property>
         <name>mapred.temp.dir</name>
         <value><%= mapred_temp_dir %></value>
@@ -99,7 +99,7 @@ mapred_temp_dir = ''
 
     <property>
         <name>mapred.hosts</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.include</value>
+        <value><%= node['hadoop']['mapred.hosts'] %></value>
         <description>
           Names a file that contains the list of nodes that may connect to the jobtracker.
           If the value is empty, all hosts are permitted.
@@ -107,17 +107,17 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapred.hosts.exclude</name>
-        <value><%= node['grid']['app_root'] %>/hadoop/conf/hosts.exclude</value>
+        <value><%= node['hadoop']['mapred.hosts.exclude'] %></value>
         <description>
           Names a file that contains the list of hosts that should be excluded by the jobtracker.
           If the value is empty, no hosts are excluded.
         </description>
     </property>
 
-<% if node['hadoop']['with_security'] then %>
+<% if node['hadoop']['with_security'] then -%>
     <property>
         <name>mapred.acls.enabled</name>
-        <value>true</value>
+        <value><%= node['hadoop']['mapred.acls.enabled'] %></value>
     </property>
     <property>
         <name>mapreduce.cluster.administrators</name>
@@ -125,13 +125,13 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.jobtracker.kerberos.principal</name>
-        <value>mapred/_HOST@${this.realm}</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.kerberos.principal'] %></value>
         <!-- _HOST is replaced with the mapred.job.tracker's host name -->
         <!-- <value>mapred/${this.jobtracker.fqdn}@${this.realm}</value> -->
     </property>
     <property>
         <name>mapreduce.jobtracker.keytab.file</name>
-        <value>${this.keytab.dir}/jt.keytab</value>
+        <value><%= node['hadoop']['mapreduce.jobtracker.keytab.file'] %></value>
     </property>
     <property>
         <name>mapreduce.tasktracker.kerberos.principal</name>
@@ -140,7 +140,7 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.tasktracker.keytab.file</name>
-        <value>${this.keytab.dir}/tt.keytab</value>
+        <value><%= node['hadoop']['mapreduce.tasktracker.keytab.file'] %></value>
     </property>
 
     <property>
@@ -155,24 +155,23 @@ mapred_temp_dir = ''
     </property>
     <property>
         <name>mapreduce.history.server.http.address</name>
-        <value>${this.jobhistory.fqdn}:19888</value>
+        <value><%= node['hadoop']['mapreduce.history.server.http.address'] %></value>
     </property>
     <property>
         <name>mapreduce.jobhistory.kerberos.principal</name>
-        <value>mapred/${this.jobhistory.fqdn}@${this.realm}</value>
+        <value><%= node['hadoop']['mapreduce.jobhistory.kerberos.principal'] %></value>
     </property>
     <property>
         <name>mapreduce.jobhistory.keytab.file</name>
-        <value>${this.keytab.dir}/jh.keytab</value>
+        <value><%= node['hadoop']['mapreduce.jobhistory.keytab.file'] %></value>
     </property>
-<% end %>
-
+<% end -%>
 <%
 this_file = 'mapred-site.xml'
 if defined? node['hadoop']['extra_configs'][this_file] \
   && node['hadoop']['extra_configs'][this_file] != nil then
   node['hadoop']['extra_configs'][this_file].each do |key,value|
-%>
+-%>
     <property>
         <name><%= key %></name>
         <value><%= value %></value>
@@ -180,5 +179,5 @@ if defined? node['hadoop']['extra_configs'][this_file] \
 <%
   end
 end 
-%>
+-%>
 </configuration>

diff --git a/cookbooks/hbase/attributes/default.rb b/cookbooks/hbase/attributes/default.rb
index 6da960e..7fe21e3 100644 (file)
--- a/cookbooks/hbase/attributes/default.rb
+++ b/cookbooks/hbase/attributes/default.rb
@@ -25,8 +25,11 @@ default['hbase']['archive_url'] = 'http://archive.apache.org/dist/hbase'
 default['hbase']['member_of_hadoop'] = false
 default['hbase']['with_security'] = false
 ## hbase-env.sh
+default['hbase']['HBASE_PID_DIR'] = "#{node['grid']['vol_root']}/0/var/run/${USER}"
+default['hbase']['HBASE_LOG_DIR'] = "#{node['grid']['vol_root']}/0/var/log/${USER}"
 default['hbase']['HBASE_PREFIX'] = "#{node['grid']['app_root']}/hbase"
 default['hbase']['HBASE_CLASSPATH'] = "#{node['hadoop']['HADOOP_PREFIX']}/conf"
+default['hbase']['HBASE_OPTS'] = '-XX:+UseConcMarkSweepGC'
 default['hbase']['HBASE_MANAGES_ZK'] = 'false'
 ## hbase-site.xml
 default['hbase']['this.cluster.name'] = 'localhost'

diff --git a/cookbooks/hbase/templates/default/conf/hbase-env.sh b/cookbooks/hbase/templates/default/conf/hbase-env.sh
index 4e55400..1be5a9c 100644 (file)
--- a/cookbooks/hbase/templates/default/conf/hbase-env.sh
+++ b/cookbooks/hbase/templates/default/conf/hbase-env.sh
@@ -29,9 +29,9 @@
 export JAVA_HOME=<%= node['java']['java_home'] %>
 
 # The directory where pid files are stored. /tmp by default.
-export HBASE_PID_DIR=<%= node['grid']['vol_root'] %>/0/var/run/${USER}
+export HBASE_PID_DIR=<%= node['hbase']['HBASE_PID_DIR'] %>
 # Where log files are stored.  $HBASE_HOME/logs by default.
-export HBASE_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/${USER}
+export HBASE_LOG_DIR=<%= node['hbase']['HBASE_LOG_DIR'] %>
 
 # export HADOOP_HOME or HADOOP_PREFIX
 # for adding hadoop native library path to java.library.path. see bin/hbase.
@@ -39,7 +39,7 @@ export HBASE_LOG_DIR=<%= node['grid']['vol_root'] %>/0/var/log/${USER}
 export HADOOP_PREFIX=<%= node['hadoop']['HADOOP_PREFIX'] %>
 export HBASE_CLASSPATH=<%= node['hbase']['HBASE_CLASSPATH'] %>
 
-export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+export HBASE_OPTS="<%= node['hbase']['HBASE_OPTS'] %>"
  
 # Tell HBase whether it should manage its own instance of Zookeeper or not.
 export HBASE_MANAGES_ZK=<%= node['hbase']['HBASE_MANAGES_ZK'] %>