update extra properties for Ptolemy hadoop cluster.

diff --git a/roles/hadoop-ptolemy.rb b/roles/hadoop-ptolemy.rb
index e14c200..a5e76af 100644 (file)
--- a/roles/hadoop-ptolemy.rb
+++ b/roles/hadoop-ptolemy.rb
@@ -30,7 +30,7 @@ hadoop_attributes = {
   'dfs.name.dir' => "#{grid_vol_root}/0/var/lib/${user.name}/name",
   #'dfs.name.dir' => "#{grid_vol_root}/0/var/lib/${user.name}/name,/export/home/${user.name}/var/lib/name",
   'dfs.replication' => '2',
-  #'dfs.replication' => '3',
+  #'dfs.replication' => '3',  # default, recommended.
   # mapred-site.xml
   'this.jobtracker.fqdn' => '${this.cluster.name}-jt.${this.domain}',
   'this.jobhistory.fqdn' => '${this.cluster.name}-jt.${this.domain}',
@@ -39,28 +39,42 @@ hadoop_attributes = {
     # e.g. 'core-site.xml' => {'k1' => 'v1', 'k2' => 'v2'},
     'hadoop-env.sh' => {
       #'HADOOP_HEAPSIZE' => '2000',               # default: 1000 MB
-      #'HADOOP_NAMENODE_INIT_HEAPSIZE' => '2000'  # default: 1000 MB (ver. 2.0)
+      #'HADOOP_NAMENODE_INIT_HEAPSIZE' => '2000'  # default: 1000 MB (ver. 2.x)
     },
-    'yarn-env.sh' => {  # ver. 2.0
+    'yarn-env.sh' => {  # ver. 2.x
       #'YARN_HEAPSIZE' => '2000'  # default: 1000 MB
     },
-    'mapred-env.sh' => {  # ver. 2.0
+    'mapred-env.sh' => {  # ver. 2.x
       #'HADOOP_JOB_HISTORYSERVER_HEAPSIZE' => '2000'  # default: 1000 MB
     },
-    'core-site.xml' => {},
+    'core-site.xml' => {
+    },
     'hdfs-site.xml' => {
+      'dfs.block.size' => '134217728',  # default: 67108864 (bytes)
+      #'dfs.blocksize' => '134217728',  # default: 134217728 (bytes) (ver. 2.x)
+      #'dfs.datanode.du.reserved' => '0',  # default: 0 (byte)
       # with HBase
       #'dfs.support.append' => 'true',
-      #'truenode.max.xcievers' => '4096'
+      #'dfs.datanode.max.xcievers' => '4096',  # default: 4096 (ver. 1.1 or later), 256 (older ver.)
+      #'dfs.datanode.max.transfer.threads' => '4096',  # default: 4096 (ver. 2.x)
+    },
+    'yarn-site.xml' => {  # ver. 2.x
+      #'yarn.nodemanager.resource.cpu-vcores' => '8',  # default: 8
+      #'yarn.nodemanager.resource.memory-mb' => '8192',  # default: 8192
     },
-    'yarn-site.xml' => {},  # ver. 2.0
     'mapred-site.xml' => {
+      # map tasks:reduce tasks = 4:3 or 2:1 or ...
       'mapred.tasktracker.map.tasks.maximum' => '4',     # default: 2
       'mapred.tasktracker.reduce.tasks.maximum' => '2',  # default: 2
       #'mapred.child.java.opts' => '-Xmx1000m',           # default: -Xmx200m
-      #'mapred.reduce.slowstart.completed.maps' => '0.80',  # default: 0.05
-      #'tasktracker.http.threads' => '80',                  # default: 40
-      #'mapred.reduce.parallel.copies' => SQRT(nodes)       # default: 5
+      'mapred.reduce.slowstart.completed.maps' => '0.80',  # default: 0.05
+      #'mapreduce.job.reduce.slowstart.completedmaps' => '0.80',  # default: 0.05 (ver. 2.x)
+      #'tasktracker.http.threads' => '80',            # default: 40
+      #'mapreduce.tasktracker.http.threads' => '80',  # default: 40 (ver. 2.x)
+      #'mapred.reduce.parallel.copies' => SQRT(nodes),            # default: 5
+      #'mapreduce.reduce.shuffle.parallelcopies' => SQRT(nodes),  # default: 5 (ver. 2.x)
+      'mapred.submit.replication' => '2',                 # default: 10
+      #'mapreduce.client.submit.file.replication' => '3',  # default: 10 (ver. 2.x)
     },
     'capacity-scheduler.xml' => {}
   }