diff options
| author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-09 20:26:31 -0700 |
|---|---|---|
| committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-09 20:26:31 -0700 |
| commit | 28c0518379d226ac25597c2840c5c81bd8551487 (patch) | |
| tree | 3414260a4e7f02cf6142f84484a62460ab3c2753 /scalding/src/main/scala/sandcrawler/CrossrefScorable.scala | |
| parent | 818ad070626d6af7c490017e0bd9b53f30f20150 (diff) | |
| download | sandcrawler-28c0518379d226ac25597c2840c5c81bd8551487.tar.gz sandcrawler-28c0518379d226ac25597c2840c5c81bd8551487.zip | |
WIP
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/CrossrefScorable.scala')
| -rw-r--r-- | scalding/src/main/scala/sandcrawler/CrossrefScorable.scala | 9 |
1 files changed, 6 insertions, 3 deletions
diff --git a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala index b221718..249c9ab 100644 --- a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala +++ b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala @@ -10,10 +10,13 @@ import parallelai.spyglass.hbase.HBasePipeConversions import parallelai.spyglass.hbase.HBaseSource class CrossrefScorable extends Scorable { - def getFeaturesPipe(args : Args) : TypedPipe[MapFeatures] = { - // TODO: Generalize args so there can be multiple Grobid pipes in one job. + // TODO: Generalize args so there can be multiple Grobid pipes in one job. + def getSource(args : Args) : Source = { TextLine(args("crossref-input")) - .read + } + + def getFeaturesPipe(pipe : Pipe) : TypedPipe[MapFeatures] = { + pipe .toTypedPipe[String](new Fields("line")) .map{ json : String => CrossrefScorable.crossrefToSlug(json) match { |