diff options
author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-06-09 10:24:18 -0700 |
---|---|---|
committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-06-09 10:24:18 -0700 |
commit | 146b3eaff0535656c71dbcb5cf1afbca1b3d33e5 (patch) | |
tree | a16b40ed6fe8ab9153df119e8c31c099ca1ba933 /scalding/src/main/scala | |
parent | 8d8ef0d8d19350228ae4d58267b156519ad1fc35 (diff) | |
download | sandcrawler-146b3eaff0535656c71dbcb5cf1afbca1b3d33e5.tar.gz sandcrawler-146b3eaff0535656c71dbcb5cf1afbca1b3d33e5.zip |
Pulled out HBaseBuilder.parseColSpec.
Diffstat (limited to 'scalding/src/main/scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/HBaseBuilder.scala | 31 | ||||
-rw-r--r-- | scalding/src/main/scala/sandcrawler/HBaseCountJob.scala | 2 |
2 files changed, 19 insertions, 14 deletions
diff --git a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala index 87cc0cb..85766d6 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseBuilder.scala @@ -13,20 +13,25 @@ object HBaseBuilder { "match0" -> List("status", "doi", "info")) val inverseSchema = {for ((k,vs) <- schema; v <-vs) yield (k + ":" + v)}.toList + // The argument should be of the form family:column, such as "file:size". + @throws(classOf[IllegalArgumentException]) + def parseColSpec(colSpec: String) { + if (!(inverseSchema contains colSpec)) { + throw new IllegalArgumentException("No such column: " + colSpec) + } + val pair = colSpec split(":") + if (pair.length != 2) { + throw new IllegalArgumentException("Bad column specifier " + colSpec + + " (specifiers should be family:name)") + } + (pair(0), pair(1)) + } + // The argument should be a comma-separated list of family:column, such as "f:c, file:size". @throws(classOf[IllegalArgumentException]) - def parseColSpec(colSpecs: List[String]) : (List[String], List[Fields]) = { + def parseColSpecs(colSpecs: List[String]) : (List[String], List[Fields]) = { // Verify that all column specifiers are legal. - for (colSpec <- colSpecs) { - if (!(inverseSchema contains colSpec)) { - throw new IllegalArgumentException("No such column: " + colSpec) - } - val pair = colSpec split(":") - if (colSpec.split(":").length != 2) { - throw new IllegalArgumentException("Bad column specifier " + colSpec + - " (specifiers should be family:name)") - } - } + for (colSpec <- colSpecs) parseColSpec(colSpec) // Produce and return a tuple containing: // 1. A list of column families. @@ -39,8 +44,8 @@ object HBaseBuilder { (families, groupedColNames.map({fields => new Fields(fields : _*)})) } - def build(table: String, server: String, colSpec: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = { - val (families, fields) = parseColSpec(colSpec) + def build(table: String, server: String, colSpecs: List[String], sourceMode: SourceMode, keyList: List[String] = List("key")) = { + val (families, fields) = parseColSpecs(colSpecs) new HBaseSource(table, server, new Fields("key"), families, fields, sourceMode = sourceMode, keyList = keyList) } } diff --git a/scalding/src/main/scala/sandcrawler/HBaseCountJob.scala b/scalding/src/main/scala/sandcrawler/HBaseCountJob.scala index 250f175..815478b 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseCountJob.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseCountJob.scala @@ -10,7 +10,7 @@ import parallelai.spyglass.hbase.HBaseConstants.SourceMode class HBaseCountJob(args: Args, colSpec: String) extends JobBase(args) with HBasePipeConversions { val output = args("output") - // TODO: Add error checking. + HBaseBuilder.parseColSpec(colSpec) val Col: String = colSpec.split(":")(1) HBaseCountJob.getHBaseSource(colSpec) |