aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
diff options
context:
space:
mode:
authorEllen Spertus <ellen.spertus@gmail.com>2018-06-03 17:16:35 -0700
committerEllen Spertus <ellen.spertus@gmail.com>2018-06-03 17:16:35 -0700
commitf2a0d6fe40be3a9b282a4cc0ea0ebdb85810762b (patch)
tree678fb58f63e7d4bda75357dec6a748f959632892 /scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
parenta9fe176a305b474050727d80c1f42941ecf1a2dc (diff)
downloadsandcrawler-f2a0d6fe40be3a9b282a4cc0ea0ebdb85810762b.tar.gz
sandcrawler-f2a0d6fe40be3a9b282a4cc0ea0ebdb85810762b.zip
Changed interface to HBaseBuilder.parseColSpec.
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/HBaseBuilder.scala')
-rw-r--r--scalding/src/main/scala/sandcrawler/HBaseBuilder.scala8
1 files changed, 6 insertions, 2 deletions
diff --git a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
index 35aeb46..eb108a2 100644
--- a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
+++ b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala
@@ -15,8 +15,12 @@ object HBaseBuilder {
// Code from https://stackoverflow.com/a/50595189/6310511
val inverseSchema = for ((k, vs) <- schema; v <- vs) yield (k + ":" + v, k)
+ // The argument should be a string with a comma-separated list of family:column,
+ // such as "f:c, file:size". Spaces after the comma are optional.
@throws(classOf[IllegalArgumentException])
- def parseColSpec(colSpecs: List[String]) : (List[String], List[Fields]) = {
+ def parseColSpec(colSpec: String) : (List[String], List[Fields]) = {
+ val colSpecs = (if (colSpec.trim.length == 0) List() else colSpec.split(", *").toList)
+
// Verify that all column specifiers are legal.
for (colSpec <- colSpecs) {
if (!(inverseSchema contains colSpec)) {
@@ -40,7 +44,7 @@ object HBaseBuilder {
(families, groupedColNames.map({fields => new Fields(fields : _*)}))
}
- def build(table: String, server: String, colSpec: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = {
+ def build(table: String, server: String, colSpec: String, sourceMode: SourceMode, keyList: List[String] = List("key")) = {
val (families, fields) = parseColSpec(colSpec)
new HBaseSource(table, server, new Fields("key"), families, fields, sourceMode = sourceMode, keyList = keyList)
}