diff options
author | Bryan Newbold <bnewbold@archive.org> | 2018-08-20 18:50:07 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2018-08-21 21:25:56 -0700 |
commit | 7c49fecd01cbd89bb5987442b89a4aafc1186ff9 (patch) | |
tree | c73d532fde19e2492fff573edb1d0ef10d7b084f /scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala | |
parent | 9514b6a3620a98e4fc069ca31b77eac6f9c98bec (diff) | |
download | sandcrawler-7c49fecd01cbd89bb5987442b89a4aafc1186ff9.tar.gz sandcrawler-7c49fecd01cbd89bb5987442b89a4aafc1186ff9.zip |
make col counter generic
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala | 37 |
1 files changed, 37 insertions, 0 deletions
diff --git a/scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala new file mode 100644 index 0000000..a007339 --- /dev/null +++ b/scalding/src/main/scala/sandcrawler/HBaseColCountJob.scala @@ -0,0 +1,37 @@ +package sandcrawler + +import java.util.Properties + +import cascading.property.AppProps +import cascading.tuple.Fields +import com.twitter.scalding._ +import parallelai.spyglass.base.JobBase +import parallelai.spyglass.hbase.HBaseConstants.SourceMode +import parallelai.spyglass.hbase.HBasePipeConversions +import parallelai.spyglass.hbase.HBaseSource + +class HBaseColCountJob(args: Args) extends JobBase(args) with HBasePipeConversions { + + val output = args("output") + + HBaseColCountJob.getHBaseSource( + args("hbase-table"), + args("zookeeper-hosts"), + args("column") + .read + .debug + .groupAll { _.size('count) } + .write(Tsv(output)) +} + +object HBaseColCountJob { + + // eg, "wbgrp-journal-extract-0-qa",7 "mtrcs-zk1.us.archive.org:2181" + def getHBaseSource(hbaseTable: String, zookeeperHosts: String, col: String) : HBaseSource = { + HBaseBuilder.build( + hbaseTable, + zookeeperHosts, + List(col), + SourceMode.SCAN_ALL) + } +} |