diff options
author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-27 18:05:21 -0700 |
---|---|---|
committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-28 07:38:56 -0700 |
commit | e18eac151d2ec2a16fc809560fb0577488c89c8c (patch) | |
tree | b7d8ca3eb90a85798f3a9af36da0eca88fe23f16 /scalding/src/main/scala/sandcrawler/BibjsonScorable.scala | |
parent | b4eac17049e19d33b1a55664a7258c0f62f8a8c7 (diff) | |
download | sandcrawler-e18eac151d2ec2a16fc809560fb0577488c89c8c.tar.gz sandcrawler-e18eac151d2ec2a16fc809560fb0577488c89c8c.zip |
replaced NoSlug with proper use of Option
Diffstat (limited to 'scalding/src/main/scala/sandcrawler/BibjsonScorable.scala')
-rw-r--r-- | scalding/src/main/scala/sandcrawler/BibjsonScorable.scala | 15 |
1 files changed, 9 insertions, 6 deletions
diff --git a/scalding/src/main/scala/sandcrawler/BibjsonScorable.scala b/scalding/src/main/scala/sandcrawler/BibjsonScorable.scala index 0d26d75..abf9220 100644 --- a/scalding/src/main/scala/sandcrawler/BibjsonScorable.scala +++ b/scalding/src/main/scala/sandcrawler/BibjsonScorable.scala @@ -15,7 +15,7 @@ class BibjsonScorable extends Scorable { TextLine(args("bibjson-input")) } - def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = { + def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[Option[MapFeatures]] = { getSource(args).read .toTypedPipe[String](new Fields("line")) .map { BibjsonScorable.bibjsonToMapFeatures(_) } @@ -23,9 +23,9 @@ class BibjsonScorable extends Scorable { } object BibjsonScorable { - def bibjsonToMapFeatures(json : String) : MapFeatures = { + def bibjsonToMapFeatures(json : String) : Option[MapFeatures] = { Scorable.jsonToMap(json) match { - case None => MapFeatures(Scorable.NoSlug, json) + case None => None case Some(map) => { if (map contains "title") { val title = Scorable.getString(map, "title") @@ -33,13 +33,16 @@ object BibjsonScorable { val sha1 = Scorable.getString(map, "sha") // TODO: year, authors (if available) if (title == null || title.isEmpty) { - new MapFeatures(Scorable.NoSlug, json) + None } else { val sf : ScorableFeatures = ScorableFeatures.create(title=title, doi=doi, sha1=sha1) - new MapFeatures(sf.toSlug, sf.toString) + sf.toSlug match { + case None => None + case Some(slug) => Some(MapFeatures(slug, sf.toString)) + } } } else { - new MapFeatures(Scorable.NoSlug, json) + None } } } |