Merge branch 'spertus-abstract' into 'master'

Refactoring to enable easy creation of groupby jobs.

See merge request webgroup/sandcrawler!6

3 files changed, 8 insertions, 8 deletions

diff --git a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
index 4697f56..603a4c7 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
@@ -4,8 +4,8 @@ import cascading.tuple.Fields
 import org.scalatest._
 
 class HBaseBuilderTest extends FlatSpec with Matchers {
-  "parseColSpec()" should "work on legal nontrivial input" in {
-    val (fams, fields) = HBaseBuilder.parseColSpec(List("file:size", "file:cdx", "match0:status"))
+  "parseColSpecs()" should "work on legal nontrivial input" in {
+    val (fams, fields) = HBaseBuilder.parseColSpecs(List("file:size", "file:cdx", "match0:status"))
     fams should have length 2
     fields should have length 2
     val fileIndex = fams.indexOf("file")
@@ -17,26 +17,26 @@ class HBaseBuilderTest extends FlatSpec with Matchers {
   }
 
   it should "work on empty input" in {
-    val (fams, fields) = HBaseBuilder.parseColSpec(List())
+    val (fams, fields) = HBaseBuilder.parseColSpecs(List())
     fams should have length 0
     fields should have length 0
   }
 
   it should "throw IllegalArgumentException on malformed input" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("file_size"))
+      HBaseBuilder.parseColSpecs(List("file_size"))
     }
   }
 
   it should "throw IllegalArgumentException on nonexistent family" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("foo:bar"))
+      HBaseBuilder.parseColSpecs(List("foo:bar"))
     }
   }
 
   it should "throw IllegalArgumentException on nonexistent column" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("file:bar"))
+      HBaseBuilder.parseColSpecs(List("file:bar"))
     }
   }
 }
diff --git a/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
index eb6f4ff..cc400c5 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
@@ -41,7 +41,7 @@ class HBaseMimeCountTest extends FunSpec with TupleConversions {
     .arg("app.conf.path", "app.conf")
     .arg("output", output)
     .arg("debug", "true")
-    .source[Tuple](HBaseMimeCountJob.getHBaseSource,
+    .source[Tuple](HBaseCountJob.getHBaseSource("file:mime"),
       sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*)))
       .sink[Tuple](Tsv(output)) {
         outputBuffer =>
diff --git a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
index 8b5c3d6..a0b313e 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
@@ -41,7 +41,7 @@ class HBaseStatusCountTest extends FunSpec with TupleConversions {
     .arg("app.conf.path", "app.conf")
     .arg("output", output)
     .arg("debug", "true")
-    .source[Tuple](HBaseStatusCountJob.getHBaseSource,
+    .source[Tuple](HBaseCountJob.getHBaseSource("grobid0:status"),
       sampleData.map(l => new Tuple(l.map(s => {new ImmutableBytesWritable(Bytes.toBytes(s))}):_*)))
       .sink[Tuple](Tsv(output)) {
         outputBuffer =>