src/main/scala/parallelai/spyglass/base/JobRunner.scala


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

package parallelai.spyglass.base

import org.apache.hadoop.conf.Configuration
import com.twitter.scalding.Tool
import org.apache.hadoop
import org.apache.hadoop.hbase.security.token.TokenUtil
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.hbase.security.User

object JobRunner {
  def main(args : Array[String]) {
    val conf: Configuration = new Configuration
    
    // TODO replace println with logging
    if (args.contains("--heapInc")) {
	    println("Setting JVM Memory/Heap Size for every child mapper and reducer.");
	    val jvmOpts = "-Xmx4096m -XX:+PrintGCDetails -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=50"
	    println("**** JVM Options : " + jvmOpts )
	    conf.set("mapred.child.java.opts", jvmOpts);
    }
     
    AppConfig.jobConfig = conf

    if (User.isHBaseSecurityEnabled(conf)) {
      println("Obtaining token for HBase security.");
      TokenUtil.obtainAndCacheToken(conf, UserGroupInformation.getCurrentUser());
    }

    hadoop.util.ToolRunner.run(conf, new Tool, args);
  }
}