diff options
author | Bryan Newbold <bnewbold@archive.org> | 2018-08-24 17:08:01 -0700 |
---|---|---|
committer | Bryan Newbold <bnewbold@archive.org> | 2018-08-24 17:08:01 -0700 |
commit | ab4ccb175621618d735b62b9f1afa764f9ad5109 (patch) | |
tree | 459b67ef402d7aaefb62e22894b02bf3be3bf902 /scalding/src/main/scala/sandcrawler/GrobidScorable.scala | |
parent | 4f1c04d6642b883be0c1d9f0e48a83932efc6411 (diff) | |
download | sandcrawler-ab4ccb175621618d735b62b9f1afa764f9ad5109.tar.gz sandcrawler-ab4ccb175621618d735b62b9f1afa764f9ad5109.zip |
clean up commented out code in scalding/
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/GrobidScorable.scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/GrobidScorable.scala | 5 |
1 files changed, 2 insertions, 3 deletions
diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala index c55cb40..899ce66 100644 --- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala +++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala @@ -16,20 +16,19 @@ class GrobidScorable extends Scorable with HBasePipeConversions { val StatusOK = 200 def getSource(args : Args) : Source = { - // TODO: Generalize args so there can be multiple grobid pipes in one job. + // TODO: Generalize args so there can be multiple grobid pipes in one job GrobidScorable.getHBaseSource(args("hbase-table"), args("zookeeper-hosts")) } def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = { getSource(args) .read - // Can't just "fromBytesWritable" because we have multiple types? + // Can't just "fromBytesWritable" because we have multiple types .toTypedPipe[(ImmutableBytesWritable,ImmutableBytesWritable,ImmutableBytesWritable)](new Fields("key", "metadata", "status_code")) .filter { case (_, metadata, status_code) => metadata != null && status_code != null } .map { case (key, metadata, status_code) => (Bytes.toString(key.copyBytes()), Bytes.toString(metadata.copyBytes()), Bytes.toLong(status_code.copyBytes())) } - // TODO: Should I combine next two stages for efficiency? .collect { case (key, json, StatusOK) => (key, json) } .filter { case (key, json) => GrobidScorable.keepRecord(json) } .map { entry : (String, String) => GrobidScorable.jsonToMapFeatures(entry._1, entry._2) } |