diff options
| author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-10 19:59:40 -0700 |
|---|---|---|
| committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-10 19:59:40 -0700 |
| commit | b7f77f6337b450406ae0a90b81faeba27394afb0 (patch) | |
| tree | dfe2ffa3a606e6b82c69f821e4415d93ab6e0676 /scalding/src/main/scala/sandcrawler/GrobidScorable.scala | |
| parent | 5ce5e5dc98cdbb5a84c79313df93d670111e6a1d (diff) | |
| download | sandcrawler-b7f77f6337b450406ae0a90b81faeba27394afb0.tar.gz sandcrawler-b7f77f6337b450406ae0a90b81faeba27394afb0.zip | |
It compiles
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/GrobidScorable.scala')
| -rw-r--r-- | scalding/src/main/scala/sandcrawler/GrobidScorable.scala | 7 |
1 files changed, 4 insertions, 3 deletions
diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala index ba15f22..61055f2 100644 --- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala +++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala @@ -16,10 +16,11 @@ class GrobidScorable extends Scorable with HBasePipeConversions { GrobidScorable.getHBaseSource(args("hbase-table"), args("zookeeper-hosts")) } - def getFeaturesPipe(pipe : cascading.pipe.Pipe) : TypedPipe[MapFeatures] = { - pipe + def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = { + getSource(args) + .read .fromBytesWritable(new Fields("key", "tei_json")) - .toTypedPipe[(String, String)](new Fields('key, 'tei_json)) + .toTypedPipe[(String, String)](new Fields("key", "tei_json")) .map { entry => val (key : String, json : String) = (entry._1, entry._2) GrobidScorable.grobidToSlug(json) match { |