aboutsummaryrefslogtreecommitdiffstats
path: root/scald-mvp/src
diff options
context:
space:
mode:
Diffstat (limited to 'scald-mvp/src')
-rw-r--r--scald-mvp/src/main/scala/sandcrawler/HBaseRowCountJob.scala23
1 files changed, 23 insertions, 0 deletions
diff --git a/scald-mvp/src/main/scala/sandcrawler/HBaseRowCountJob.scala b/scald-mvp/src/main/scala/sandcrawler/HBaseRowCountJob.scala
new file mode 100644
index 0000000..734abaa
--- /dev/null
+++ b/scald-mvp/src/main/scala/sandcrawler/HBaseRowCountJob.scala
@@ -0,0 +1,23 @@
+package sandcrawler
+
+import com.twitter.scalding._
+import parallelai.spyglass.hbase.{HBaseSource, HBasePipeConversions, HBaseConstants}
+
+class HBaseRowCountJob(args: Args) extends Job(args) {
+
+ // For now doesn't actually count, just dumps a "word count"
+
+ val hbs = new HBaseSource(
+ "wbgrp-journal-extract-0-qa", // HBase Table Name
+ "mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?)
+ 'key, // ... then a list of column names
+ sourceMode = HBaseConstants.SourceMode.SCAN_ALL)
+/*
+ .read
+ .map { word => (word, 1L) }
+ .sumByKey
+ .write(TypedTsv[(String, Long)](args("output")))
+ // The compiler will enforce the type coming out of the sumByKey is the same as the type we have for our sink
+ .flatMap { line => line.split("\\s+") }
+*/
+}