From f2a0d6fe40be3a9b282a4cc0ea0ebdb85810762b Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Sun, 3 Jun 2018 17:16:35 -0700
Subject: Changed interface to HBaseBuilder.parseColSpec.

---
 scalding/src/main/scala/sandcrawler/HBaseBuilder.scala     |  8 ++++++--
 scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala |  2 +-
 scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala | 10 +++++-----
 3 files changed, 12 insertions(+), 8 deletions(-)

(limited to 'scalding')

diff --git a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
index 35aeb46..eb108a2 100644
--- a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
+++ b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
@@ -15,8 +15,12 @@ object HBaseBuilder {
   // Code from https://stackoverflow.com/a/50595189/6310511
   val inverseSchema = for ((k, vs) <- schema; v <- vs) yield (k + ":" + v, k)
 
+  // The argument should be a string with a comma-separated list of family:column, 
+  // such as "f:c, file:size". Spaces after the comma are optional.
   @throws(classOf[IllegalArgumentException])
-  def parseColSpec(colSpecs: List[String]) : (List[String], List[Fields]) = {
+  def parseColSpec(colSpec: String) : (List[String], List[Fields]) = {
+    val colSpecs = (if (colSpec.trim.length == 0) List() else colSpec.split(", *").toList)
+
     // Verify that all column specifiers are legal.
     for (colSpec <- colSpecs) {
       if (!(inverseSchema contains colSpec)) {
@@ -40,7 +44,7 @@ object HBaseBuilder {
     (families, groupedColNames.map({fields => new Fields(fields : _*)}))
   }
 
-  def build(table: String, server: String, colSpec: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = {
+  def build(table: String, server: String, colSpec: String, sourceMode: SourceMode, keyList: List[String] = List("key")) = {
     val (families, fields) = parseColSpec(colSpec)
     new HBaseSource(table, server, new Fields("key"), families, fields, sourceMode = sourceMode, keyList = keyList)
   }
diff --git a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
index 9d074ab..041df4d 100644
--- a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
+++ b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
@@ -27,6 +27,6 @@ object HBaseRowCountJob {
   def getHBaseSource = HBaseBuilder.build(
     "wbgrp-journal-extract-0-qa",     // HBase Table Name
     "mtrcs-zk1.us.archive.org:2181",  // HBase Zookeeper server (to get runtime config info; can be array?)
-    List("file:size", "file:mime"),
+    "file:size, file:mime",
     SourceMode.SCAN_ALL)
 }
diff --git a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
index b45751d..8392100 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
@@ -6,7 +6,7 @@ import sandcrawler.HBaseBuilder
 
 class HBaseBuilderTest extends FlatSpec with Matchers {
   "parseColSpec()" should "work on legal nontrivial input" in {
-    val (fams, fields) = HBaseBuilder.parseColSpec(List("file:size", "file:cdx", "match0:status"))
+    val (fams, fields) = HBaseBuilder.parseColSpec("file:size, file:cdx, match0:status")
     fams should have length 2
     fields should have length 2
     val fileIndex = fams.indexOf("file")
@@ -18,26 +18,26 @@ class HBaseBuilderTest extends FlatSpec with Matchers {
   }
 
   it should "work on empty input" in {
-    val (fams, fields) = HBaseBuilder.parseColSpec(List())
+    val (fams, fields) = HBaseBuilder.parseColSpec("")
     fams should have length 0
     fields should have length 0
   }
 
   it should "throw IllegalArgumentException on malformed input" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("file_size"))
+      HBaseBuilder.parseColSpec("file_size")
     }
   }
 
   it should "throw IllegalArgumentException on nonexistent family" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("foo:bar"))
+      HBaseBuilder.parseColSpec("foo:bar")
     }
   }
 
   it should "throw IllegalArgumentException on nonexistent column" in {
     a [IllegalArgumentException] should be thrownBy {
-      HBaseBuilder.parseColSpec(List("file:bar"))
+      HBaseBuilder.parseColSpec("file:bar")
     }
   }
 }
-- 
cgit v1.2.3