From 3e9b927ba668f13f818d20962a6fb6f5783407df Mon Sep 17 00:00:00 2001 From: Ellen Spertus Date: Sat, 9 Jun 2018 10:38:04 -0700 Subject: Refactored HBaseStatusCount{Job,Test}.scala. --- .../scala/sandcrawler/HBaseStatusCountJob.scala | 27 ++-------------------- .../scala/sandcrawler/HBaseStatusCountTest.scala | 2 +- 2 files changed, 3 insertions(+), 26 deletions(-) (limited to 'scalding/src') diff --git a/scalding/src/main/scala/sandcrawler/HBaseStatusCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseStatusCountJob.scala index 0675efc..27b3177 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseStatusCountJob.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseStatusCountJob.scala @@ -1,28 +1,5 @@ package sandcrawler -import cascading.property.AppProps -import cascading.tuple.Fields -import com.twitter.scalding._ -import java.util.Properties -import parallelai.spyglass.base.JobBase -import parallelai.spyglass.hbase.{HBaseSource, HBasePipeConversions} -import parallelai.spyglass.hbase.HBaseConstants.SourceMode +import com.twitter.scalding.Args -class HBaseStatusCountJob(args: Args) extends JobBase(args) with HBasePipeConversions { - val output = args("output") - - HBaseStatusCountJob.getHBaseSource - .read - .fromBytesWritable(List('status)) -// .debug - .groupBy('status){group => group.size('count)} - .write(Tsv(output)) -} - -object HBaseStatusCountJob { - def getHBaseSource = HBaseBuilder.build( - "wbgrp-journal-extract-0-qa", // HBase Table Name - "mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?) - List("grobid0:status"), - SourceMode.SCAN_ALL) -} +class HBaseStatusCountJob(args: Args) extends HBaseCountJob(args, "grobid0:status") diff --git a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala index 8b5c3d6..a0b313e 100644 --- a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala +++ b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala @@ -41,7 +41,7 @@ class HBaseStatusCountTest extends FunSpec with TupleConversions { .arg("app.conf.path", "app.conf") .arg("output", output) .arg("debug", "true") - .source[Tuple](HBaseStatusCountJob.getHBaseSource, + .source[Tuple](HBaseCountJob.getHBaseSource("grobid0:status"), sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*))) .sink[Tuple](Tsv(output)) { outputBuffer => -- cgit v1.2.3