From dae965840db388c53b969d76849e5e8e9569ceee Mon Sep 17 00:00:00 2001 From: Ellen Spertus Date: Tue, 24 Jul 2018 12:25:45 -0700 Subject: Changed return type of grobidToSlug() to Option[String]. --- .../main/scala/sandcrawler/HBaseCrossrefScoreJob.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'scalding/src/main') diff --git a/scalding/src/main/scala/sandcrawler/HBaseCrossrefScoreJob.scala b/scalding/src/main/scala/sandcrawler/HBaseCrossrefScoreJob.scala index a22af81..d3e78fe 100644 --- a/scalding/src/main/scala/sandcrawler/HBaseCrossrefScoreJob.scala +++ b/scalding/src/main/scala/sandcrawler/HBaseCrossrefScoreJob.scala @@ -22,7 +22,7 @@ class HBaseCrossrefScoreJob(args: Args) extends JobBase(args) with HBasePipeConv sourceMode = SourceMode.SCAN_ALL) val grobidPipe = grobidSource - .read + .read .map('tei_json -> 'slug) { json : String => HBaseCrossrefScore.grobidToSlug(json)} @@ -42,17 +42,21 @@ class HBaseCrossrefScoreJob(args: Args) extends JobBase(args) with HBasePipeConv } object HBaseCrossrefScore { - def grobidToSlug(json : String) = { + def grobidToSlug(json : String) : Option[String] = { // https://stackoverflow.com/a/32717262/631051 val jsonObject = JSON.parseFull(json) - val globalMap = jsonObject.get.asInstanceOf[Map[String, Any]] - globalMap.get("title") match { - case Some(title) => titleToSlug(title.asInstanceOf[String]) - case None => "" + if (jsonObject == None) { + None + } else { + val globalMap = jsonObject.get.asInstanceOf[Map[String, Any]] + globalMap.get("title") match { + case Some(title) => Some(titleToSlug(title.asInstanceOf[String])) + case None => None + } } } - def titleToSlug(title : String) = { + def titleToSlug(title : String) : String = { title.split(":")(0) } } -- cgit v1.2.3