Created static factory method for ScorableCreations to deal with null.

4 files changed, 26 insertions, 18 deletions

diff --git a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
index ff8201a..5d1eaf5 100644
--- a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
@@ -35,7 +35,7 @@ object CrossrefScorable {
             new MapFeatures(Scorable.NoSlug, json)
           } else {
             // bnewbold: not checking that titles(0) is non-null/non-empty; case would be, in JSON, "title": [ null ]
-            val sf : ScorableFeatures = new ScorableFeatures(title=titles(0), doi=doi)
+            val sf : ScorableFeatures = ScorableFeatures.create(title=titles(0), doi=doi)
             new MapFeatures(sf.toSlug, sf.toString)
           }
         } else {
diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
index 9a09e05..d7a1eea 100644
--- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
@@ -45,7 +45,7 @@ object GrobidScorable {
       case None => MapFeatures(Scorable.NoSlug, json)
       case Some(map) => {
         if (map contains "title") {
-          new ScorableFeatures(Scorable.getString(map, "title"), sha1=key).toMapFeatures
+          ScorableFeatures.create(title=Scorable.getString(map, "title"), sha1=key).toMapFeatures
         } else {
           MapFeatures(Scorable.NoSlug, json)
         }
diff --git a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
index 8ed3369..d9461e7 100644
--- a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
+++ b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
@@ -2,12 +2,20 @@ package sandcrawler
 
 import scala.util.parsing.json.JSONObject
 
+object ScorableFeatures {
+  def create(title : String, year : Int = 0, doi : String = "", sha1 : String = "") : ScorableFeatures = {
+    new ScorableFeatures(
+      title=if (title == null) "" else title,
+      year=year,
+      doi=if (doi == null) "" else doi,
+      sha1=if (sha1 == null) "" else sha1)
+  }
+}
 
 // Contains features needed to make slug and to score (in combination
-// with a second ScorableFeatures).
-class ScorableFeatures(title : String, year: Int = 0, doi : String = "", sha1: String = "") {
-
-  val slugBlacklist = Set( "abbreviations", "abstract", "acknowledgements",
+// with a second ScorableFeatures). Create with above static factory method.
+class ScorableFeatures private(title : String, year: Int = 0, doi : String = "", sha1: String = "") {
+  val SlugBlacklist = Set( "abbreviations", "abstract", "acknowledgements",
     "article", "authorreply", "authorsreply", "bookreview", "bookreviews",
     "casereport", "commentary", "commentaryon", "commenton", "commentto",
     "contents", "correspondence", "dedication", "editorialadvisoryboard",
@@ -16,15 +24,15 @@ class ScorableFeatures(title : String, year: Int = 0, doi : String = "", sha1: S
     "references", "results", "review", "reviewarticle", "summary", "title",
     "name")
 
-  def toMap() : Map[String, Any] = {
-    Map("title" -> (if (title == null) "" else title),
-        "year" -> year,
-        "doi" -> (if (doi == null) "" else doi),
-        "sha1" -> (if (sha1 == null) "" else sha1))
-  }
+  def toMap() : Map[String, Any] = Map("title" -> title, "year" -> year, "doi" -> doi, "sha1" -> sha1)
 
   override def toString() : String = {
-    JSONObject(toMap()).toString
+    val myMap = toMap()
+    assert(myMap("title") != null)
+    assert(myMap("year") != null)
+    assert(myMap("doi") != null)
+    assert(myMap("sha1") != null)
+    JSONObject(myMap).toString
   }
 
   def toSlug() : String = {
@@ -34,11 +42,11 @@ class ScorableFeatures(title : String, year: Int = 0, doi : String = "", sha1: S
       val unaccented = StringUtilities.removeAccents(title)
       // Remove punctuation
       val slug = StringUtilities.removePunctuation((unaccented.toLowerCase())).replaceAll("\\s", "")
-      if (slug.isEmpty || slug == null || (slugBlacklist contains slug)) Scorable.NoSlug else slug
+      if (slug.isEmpty || slug == null || (SlugBlacklist contains slug)) Scorable.NoSlug else slug
     }
   }
 
-  def toMapFeatures = {
+  def toMapFeatures : MapFeatures = {
     MapFeatures(toSlug, toString)
   }
 }
diff --git a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
index 80d92aa..8a293fe 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
@@ -5,12 +5,12 @@ import org.scalatest._
 // scalastyle:off null
 class ScorableFeaturesTest extends FlatSpec with Matchers {
   private def titleToSlug(s : String) : String = {
-    new ScorableFeatures(title = s).toSlug
+    ScorableFeatures.create(title = s).toSlug
   }
 
   "toMapFeatures()" should "work with gnarly inputs" in {
-    new ScorableFeatures(title = null).toMapFeatures
-    new ScorableFeatures(title = "something", doi = null, sha1 = null, year = 123).toMapFeatures
+    ScorableFeatures.create(title = null).toMapFeatures
+    ScorableFeatures.create(title = "something", doi = null, sha1 = null, year = 123).toMapFeatures
   }
 
   "mapToSlug()" should "extract the parts of titles before a colon" in {