aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src
diff options
context:
space:
mode:
Diffstat (limited to 'scalding/src')
-rw-r--r--scalding/src/main/scala/sandcrawler/HBaseBuilder.scala13
-rw-r--r--scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala2
-rw-r--r--scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala10
3 files changed, 10 insertions, 15 deletions
diff --git a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
index eb108a2..87cc0cb 100644
--- a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
+++ b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
@@ -11,16 +11,11 @@ object HBaseBuilder {
"file" -> List("size", "mime", "cdx"),
"grobid0" -> List("status_code", "quality", "status", "tei_xml", "tei_json", "metadata"),
"match0" -> List("status", "doi", "info"))
- // map from colFamily:colName -> colFamily
- // Code from https://stackoverflow.com/a/50595189/6310511
- val inverseSchema = for ((k, vs) <- schema; v <- vs) yield (k + ":" + v, k)
+ val inverseSchema = {for ((k,vs) <- schema; v <-vs) yield (k + ":" + v)}.toList
- // The argument should be a string with a comma-separated list of family:column,
- // such as "f:c, file:size". Spaces after the comma are optional.
+ // The argument should be a comma-separated list of family:column, such as "f:c, file:size".
@throws(classOf[IllegalArgumentException])
- def parseColSpec(colSpec: String) : (List[String], List[Fields]) = {
- val colSpecs = (if (colSpec.trim.length == 0) List() else colSpec.split(", *").toList)
-
+ def parseColSpec(colSpecs: List[String]) : (List[String], List[Fields]) = {
// Verify that all column specifiers are legal.
for (colSpec <- colSpecs) {
if (!(inverseSchema contains colSpec)) {
@@ -44,7 +39,7 @@ object HBaseBuilder {
(families, groupedColNames.map({fields => new Fields(fields : _*)}))
}
- def build(table: String, server: String, colSpec: String, sourceMode: SourceMode, keyList: List[String] = List("key")) = {
+ def build(table: String, server: String, colSpec: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = {
val (families, fields) = parseColSpec(colSpec)
new HBaseSource(table, server, new Fields("key"), families, fields, sourceMode = sourceMode, keyList = keyList)
}
diff --git a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
index 041df4d..9d074ab 100644
--- a/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
+++ b/scalding/src/main/scala/sandcrawler/HBaseRowCountJob.scala
@@ -27,6 +27,6 @@ object HBaseRowCountJob {
def getHBaseSource = HBaseBuilder.build(
"wbgrp-journal-extract-0-qa", // HBase Table Name
"mtrcs-zk1.us.archive.org:2181", // HBase Zookeeper server (to get runtime config info; can be array?)
- "file:size, file:mime",
+ List("file:size", "file:mime"),
SourceMode.SCAN_ALL)
}
diff --git a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
index 8392100..b45751d 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
@@ -6,7 +6,7 @@ import sandcrawler.HBaseBuilder
class HBaseBuilderTest extends FlatSpec with Matchers {
"parseColSpec()" should "work on legal nontrivial input" in {
- val (fams, fields) = HBaseBuilder.parseColSpec("file:size, file:cdx, match0:status")
+ val (fams, fields) = HBaseBuilder.parseColSpec(List("file:size", "file:cdx", "match0:status"))
fams should have length 2
fields should have length 2
val fileIndex = fams.indexOf("file")
@@ -18,26 +18,26 @@ class HBaseBuilderTest extends FlatSpec with Matchers {
}
it should "work on empty input" in {
- val (fams, fields) = HBaseBuilder.parseColSpec("")
+ val (fams, fields) = HBaseBuilder.parseColSpec(List())
fams should have length 0
fields should have length 0
}
it should "throw IllegalArgumentException on malformed input" in {
a [IllegalArgumentException] should be thrownBy {
- HBaseBuilder.parseColSpec("file_size")
+ HBaseBuilder.parseColSpec(List("file_size"))
}
}
it should "throw IllegalArgumentException on nonexistent family" in {
a [IllegalArgumentException] should be thrownBy {
- HBaseBuilder.parseColSpec("foo:bar")
+ HBaseBuilder.parseColSpec(List("foo:bar"))
}
}
it should "throw IllegalArgumentException on nonexistent column" in {
a [IllegalArgumentException] should be thrownBy {
- HBaseBuilder.parseColSpec("file:bar")
+ HBaseBuilder.parseColSpec(List("file:bar"))
}
}
}