From 9a9f51c9eea317f5d28e6bc2f930b45159e7bb57 Mon Sep 17 00:00:00 2001 From: Chandan Rajah Date: Wed, 7 Aug 2013 11:37:13 +0100 Subject: Added testing code for multi input splits --- .../spyglass/hbase/example/HBaseExample.scala | 23 --------------- .../hbase/example/HBaseExampleRunner.scala | 33 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 23 deletions(-) (limited to 'src/main/scala/parallelai') diff --git a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala index 2aa5342..13c75d6 100644 --- a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala +++ b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala @@ -30,29 +30,6 @@ class HBaseExample(args: Args) extends JobBase(args) { println("Output : " + output) println("Quorum : " + quorumNames) - case class HBaseTableStore( - conf: Configuration, - quorum: String, - tableName: String) { - - val tableBytes = Bytes.toBytes(tableName) - val connection = HConnectionManager.getConnection(conf) - val maxThreads = conf.getInt("hbase.htable.threads.max", 1) - - conf.set("hbase.zookeeper.quorum", quorumNames) - - val htable = new HTable(HBaseConfiguration.create(conf), tableName) - - def makeN(n: Int) { - (0 to n - 1).map(x => "%015d".format(x.toLong)).foreach(x => { - val put = new Put(HBaseSalter.addSaltPrefix(Bytes.toBytes(x))) - put.add(Bytes.toBytes("data"), Bytes.toBytes("data"), Bytes.toBytes(x)) - }) - } - - } - - HBaseTableStore(jobConf, quorumNames, "_TEST.SALT.01").makeN(100000) val hbs2 = new HBaseSource( "_TEST.SALT.01", diff --git a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala index 920f17d..c503247 100644 --- a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala +++ b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala @@ -4,6 +4,13 @@ import com.twitter.scalding.Tool import org.joda.time.format.DateTimeFormat import java.util.Formatter.DateTime import parallelai.spyglass.base.JobRunner +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.hbase.{HColumnDescriptor, HTableDescriptor, HBaseConfiguration} +import org.apache.hadoop.hbase.client.{Put, HTable, HConnectionManager, HBaseAdmin} +import org.apache.hadoop.hbase.io.hfile.Compression +import org.apache.hadoop.hbase.regionserver.StoreFile +import org.apache.hadoop.hbase.util.Bytes +import parallelai.spyglass.hbase.HBaseSalter object HBaseExampleRunner extends App { val appPath = System.getenv("BIGDATA_APPCONF_PATH") @@ -35,7 +42,33 @@ object HBaseExampleRunner extends App { val output = "HBaseTest.%s" + case class HBaseTableStore( + conf: Configuration, + quorum: String, + tableName: String) { + + val tableBytes = Bytes.toBytes(tableName) + val connection = HConnectionManager.getConnection(conf) + val maxThreads = conf.getInt("hbase.htable.threads.max", 1) + + conf.set("hbase.zookeeper.quorum", quorum) + + val htable = new HTable(HBaseConfiguration.create(conf), tableName) + + def makeN(n: Int) { + (0 to n - 1).map(x => "%015d".format(x.toLong)).foreach(x => { + val put = new Put(HBaseSalter.addSaltPrefix(Bytes.toBytes(x))) + put.add(Bytes.toBytes("data"), Bytes.toBytes("data"), Bytes.toBytes(x)) + }) + } + + } + + val conf: Configuration = HBaseConfiguration.create + HBaseTableStore(conf, quorum, "_TEST.SALT.01").makeN(100000) + JobRunner.main(Array(classOf[HBaseExample].getName, "--hdfs", "--app.conf.path", appPath, "--output", output, "--debug", "true", "--job.lib.path", jobLibPath, "--quorum", quorum)) + } \ No newline at end of file -- cgit v1.2.3