diff options
Diffstat (limited to 'scalding')
3 files changed, 12 insertions, 8 deletions
diff --git a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala index 35aeb46..eb108a2 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala @@ -15,8 +15,12 @@ object HBaseBuilder { // Code from https://stackoverflow.com/a/50595189/6310511 val inverseSchema = for ((k, vs) <- schema; v <- vs) yield (k + ":" + v, k) + // The argument should be a string with a comma-separated list of family:column, + // such as "f:c, file:size". Spaces after the comma are optional. @throws(classOf[IllegalArgumentException]) - def parseColSpec(colSpecs: List[String]) : (List[String], List[Fields]) = { + def parseColSpec(colSpec: String) : (List[String], List[Fields]) = { + val colSpecs = (if (colSpec.trim.length == 0) List() else colSpec.split(", *").toList) + // Verify that all column specifiers are legal. for (colSpec <- colSpecs) { if (!(inverseSchema contains colSpec)) { @@ -40,7 +44,7 @@ object HBaseBuilder { (families, groupedColNames.map({fields => new Fields(fields : _*)})) } - def build(table: String, server: String, colSpec: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = { + def build(table: String, server: String, colSpec: String, sourceMode: SourceMode, keyList: List[String] = List("key")) = { val (families, fields) = parseColSpec(colSpec) new HBaseSource(table, server, new Fields("key"), families, fields, sourceMode = sourceMode, keyList = keyList) } diff --git a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala index 9d074ab..041df4d 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala @@ -27,6 +27,6 @@ object HBaseRowCountJob { def getHBaseSource = HBaseBuilder.build( "wbgrp-journal-extract-0-qa", // HBase Table Name "mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?) - List("file:size", "file:mime"), + "file:size, file:mime", SourceMode.SCAN_ALL) } diff --git a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala index b45751d..8392100 100644 --- a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala +++ b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala @@ -6,7 +6,7 @@ import sandcrawler.HBaseBuilder class HBaseBuilderTest extends FlatSpec with Matchers { "parseColSpec()" should "work on legal nontrivial input" in { - val (fams, fields) = HBaseBuilder.parseColSpec(List("file:size", "file:cdx", "match0:status")) + val (fams, fields) = HBaseBuilder.parseColSpec("file:size, file:cdx, match0:status") fams should have length 2 fields should have length 2 val fileIndex = fams.indexOf("file") @@ -18,26 +18,26 @@ class HBaseBuilderTest extends FlatSpec with Matchers { } it should "work on empty input" in { - val (fams, fields) = HBaseBuilder.parseColSpec(List()) + val (fams, fields) = HBaseBuilder.parseColSpec("") fams should have length 0 fields should have length 0 } it should "throw IllegalArgumentException on malformed input" in { a [IllegalArgumentException] should be thrownBy { - HBaseBuilder.parseColSpec(List("file_size")) + HBaseBuilder.parseColSpec("file_size") } } it should "throw IllegalArgumentException on nonexistent family" in { a [IllegalArgumentException] should be thrownBy { - HBaseBuilder.parseColSpec(List("foo:bar")) + HBaseBuilder.parseColSpec("foo:bar") } } it should "throw IllegalArgumentException on nonexistent column" in { a [IllegalArgumentException] should be thrownBy { - HBaseBuilder.parseColSpec(List("file:bar")) + HBaseBuilder.parseColSpec("file:bar") } } } |