Added testing code for multi input splits

2 files changed, 33 insertions, 23 deletions

diff --git a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala
index 2aa5342..13c75d6 100644
--- a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala
+++ b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExample.scala
@@ -30,29 +30,6 @@ class HBaseExample(args: Args) extends JobBase(args) {
    println("Output : " + output)
    println("Quorum : " + quorumNames)
 
-   case class HBaseTableStore(
-      conf: Configuration,
-      quorum: String,
-      tableName: String) {
-
-      val tableBytes = Bytes.toBytes(tableName)
-      val connection = HConnectionManager.getConnection(conf)
-      val maxThreads = conf.getInt("hbase.htable.threads.max", 1)
-
-      conf.set("hbase.zookeeper.quorum", quorumNames)
-
-      val htable = new HTable(HBaseConfiguration.create(conf), tableName)
-
-      def makeN(n: Int) {
-         (0 to n - 1).map(x => "%015d".format(x.toLong)).foreach(x => {
-            val put = new Put(HBaseSalter.addSaltPrefix(Bytes.toBytes(x)))
-            put.add(Bytes.toBytes("data"), Bytes.toBytes("data"), Bytes.toBytes(x))
-         })
-      }
-
-   }
-
-   HBaseTableStore(jobConf, quorumNames, "_TEST.SALT.01").makeN(100000)
 
    val hbs2 = new HBaseSource(
       "_TEST.SALT.01",
diff --git a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala
index 920f17d..c503247 100644
--- a/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala
+++ b/src/main/scala/parallelai/spyglass/hbase/example/HBaseExampleRunner.scala
@@ -4,6 +4,13 @@ import com.twitter.scalding.Tool
 import org.joda.time.format.DateTimeFormat
 import java.util.Formatter.DateTime
 import parallelai.spyglass.base.JobRunner
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.{HColumnDescriptor, HTableDescriptor, HBaseConfiguration}
+import org.apache.hadoop.hbase.client.{Put, HTable, HConnectionManager, HBaseAdmin}
+import org.apache.hadoop.hbase.io.hfile.Compression
+import org.apache.hadoop.hbase.regionserver.StoreFile
+import org.apache.hadoop.hbase.util.Bytes
+import parallelai.spyglass.hbase.HBaseSalter
 
 object HBaseExampleRunner extends App {
    val appPath = System.getenv("BIGDATA_APPCONF_PATH")
@@ -35,7 +42,33 @@ object HBaseExampleRunner extends App {
 
    val output = "HBaseTest.%s"
 
+  case class HBaseTableStore(
+                              conf: Configuration,
+                              quorum: String,
+                              tableName: String) {
+
+    val tableBytes = Bytes.toBytes(tableName)
+    val connection = HConnectionManager.getConnection(conf)
+    val maxThreads = conf.getInt("hbase.htable.threads.max", 1)
+
+    conf.set("hbase.zookeeper.quorum", quorum)
+
+    val htable = new HTable(HBaseConfiguration.create(conf), tableName)
+
+    def makeN(n: Int) {
+      (0 to n - 1).map(x => "%015d".format(x.toLong)).foreach(x => {
+        val put = new Put(HBaseSalter.addSaltPrefix(Bytes.toBytes(x)))
+        put.add(Bytes.toBytes("data"), Bytes.toBytes("data"), Bytes.toBytes(x))
+      })
+    }
+
+  }
+
+  val conf: Configuration = HBaseConfiguration.create
+  HBaseTableStore(conf, quorum, "_TEST.SALT.01").makeN(100000)
+
    JobRunner.main(Array(classOf[HBaseExample].getName, "--hdfs", "--app.conf.path", appPath,
       "--output", output, "--debug", "true", "--job.lib.path", jobLibPath, "--quorum", quorum))
 
+
 }
\ No newline at end of file