From 8dc3bf5c6f68d1fffa9f940ba1024ed95e76ed64 Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Tue, 7 Aug 2018 11:05:23 -0700
Subject: Added GrobidScorableTest, minor improvements.

---
 .../scala/sandcrawler/GrobidScorableTest.scala     | 77 ++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala

(limited to 'scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala')

diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
new file mode 100644
index 0000000..7777610
--- /dev/null
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -0,0 +1,77 @@
+package sandcrawler
+
+import cascading.tuple.Fields
+import cascading.tuple.Tuple
+import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.util.Bytes
+import org.scalatest._
+import parallelai.spyglass.hbase.HBaseConstants.SourceMode
+
+class GrobidScorableTest extends FlatSpec with Matchers {
+  val GrobidString = """
+{
+  "title": "<<TITLE>>",
+  "authors": [
+    {"name": "Brewster Kahle"},
+    {"name": "J Doe"}
+  ],
+  "journal": {
+    "name": "Dummy Example File. Journal of Fake News. pp. 1-2. ISSN 1234-5678",
+    "eissn": null,
+    "issn": null,
+    "issue": null,
+    "publisher": null,
+    "volume": null
+  },
+  "date": "2000",
+  "doi": null,
+  "citations": [
+    { "authors": [{"name": "A Seaperson"}],
+      "date": "2001",
+      "id": "b0",
+      "index": 0,
+      "issue": null,
+      "journal": "Letters in the Alphabet",
+      "publisher": null,
+      "title": "Everything is Wonderful",
+      "url": null,
+      "volume": "20"},
+    { "authors": [],
+      "date": "2011-03-28",
+      "id": "b1",
+      "index": 1,
+      "issue": null,
+      "journal": "The Dictionary",
+      "publisher": null,
+      "title": "All about Facts",
+      "url": null,
+      "volume": "14"}
+  ],
+  "abstract": "Everything you ever wanted to know about nothing",
+  "body": "Introduction \nEverything starts somewhere, as somebody [1]  once said. \n\n In Depth \n Meat \nYou know, for kids. \n Potatos \nQED.",
+  "acknowledgement": null,
+  "annex": null
+}
+"""
+  val GrobidStringWithTitle = GrobidString.replace("<<TITLE>>", "Dummy Example File")
+  val GrobidStringWithoutTitle = GrobidString.replace("title", "nottitle")
+  val MalformedGrobidString = GrobidString.replace("}", "")
+
+  // Unit tests
+
+  "grobidToSlug()" should "get the right slug for a grobid json string" in {
+    val slug = GrobidScorable.grobidToSlug(GrobidStringWithTitle)
+    slug should contain ("dummy example file")
+  }
+
+  it should "return None if given json string without title" in {
+    val slug = GrobidScorable.grobidToSlug(GrobidStringWithoutTitle)
+    slug shouldBe None
+  }
+
+  it should "return None if given a malformed json string" in {
+    val slug = GrobidScorable.grobidToSlug(MalformedGrobidString)
+    slug shouldBe None
+  }
+}
-- 
cgit v1.2.3


From 728e50a33cec921c9a624439f2e1c8561a6e12ce Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Sat, 11 Aug 2018 21:03:53 -0700
Subject: It compiles.

---
 .../main/scala/sandcrawler/CrossrefScorable.scala  | 54 ++++++++++++++--------
 .../main/scala/sandcrawler/GrobidScorable.scala    | 21 ++++-----
 scalding/src/main/scala/sandcrawler/Scorable.scala | 40 +++++++++++-----
 .../scala/sandcrawler/CrossrefScorableTest.scala   | 26 ++++++-----
 .../scala/sandcrawler/GrobidScorableTest.scala     | 19 ++++----
 5 files changed, 96 insertions(+), 64 deletions(-)

(limited to 'scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala')

diff --git a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
index b2f6537..5113b0c 100644
--- a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
@@ -18,6 +18,7 @@ import java.util.regex.Pattern
 
 import scala.math
 import scala.util.parsing.json.JSON
+import scala.util.parsing.json.JSONObject
 
 import cascading.tuple.Fields
 import com.twitter.scalding._
@@ -40,33 +41,48 @@ class CrossrefScorable extends Scorable with HBasePipeConversions {
   def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = {
     getSource(args).read
       .toTypedPipe[String](new Fields("line"))
-      .map{ json : String => 
-        CrossrefScorable.simplifyJson(json) match {
-          case None => new MapFeatures(Scorable.NoSlug, json)
-          case Some(map) => new MapFeatures(
-            Scorable.titleToSlug(map("title").asInstanceOf[String]), 
-            JSONObject(map).toString)
+      .map{ json : String =>
+        Scorable.jsonToMap(json) match {
+          case None => MapFeatures(Scorable.NoSlug, json)
+          case Some(map) => {
+            if ((map contains "title") && (map contains "DOI")) {
+              val titles = map("title").asInstanceOf[List[String]]
+              if (titles.isEmpty) {
+                new MapFeatures(Scorable.NoSlug, json)
+              } else {
+                val title = titles(0)
+                val map2 = Scorable.toScorableMap(title=titles(0), doi=map("DOI").asInstanceOf[String])
+                new MapFeatures(
+                  Scorable.mapToSlug(map2),
+                  JSONObject(map2).toString)
+              }
+            } else {
+              new MapFeatures(Scorable.NoSlug, json)
+            }
+          }
         }
       }
   }
+}
 
-  object CrossrefScorable {
-    def simplifyJson(json : String) : Option[Map[String, Any]] = {
-      Scorable.jsonToMap(json) match {
-        case None => None
-        case Some(map) => {
-          if (map contains "title") {
-            val titles = map("title").asInstanceOf[List[String]]
-            if (titles.isEmpty) {
-              None
-            } else {
-              Some(Map("title" -> titles(0)))
-            }
-          } else {
+/*
+object CrossrefScorable {
+  def simplifyJson(json : String) : Option[Map[String, Any]] = {
+    Scorable.jsonToMap(json) match {
+      case None => None
+      case Some(map) => {
+        if (map contains "title") {
+          val titles = map("title").asInstanceOf[List[String]]
+          if (titles.isEmpty) {
             None
+          } else {
+            Some(Map("title" -> titles(0)))
           }
+        } else {
+          None
         }
       }
     }
   }
 }
+ */
diff --git a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
index 61055f2..de9f51a 100644
--- a/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/GrobidScorable.scala
@@ -1,5 +1,6 @@
 package sandcrawler
 
+import scala.util.parsing.json.JSONObject
 import cascading.flow.FlowDef
 import cascading.pipe.Pipe
 import cascading.tuple.Fields
@@ -21,13 +22,7 @@ class GrobidScorable extends Scorable with HBasePipeConversions {
       .read
       .fromBytesWritable(new Fields("key", "tei_json"))
       .toTypedPipe[(String, String)](new Fields("key", "tei_json"))
-      .map { entry =>
-        val (key : String, json : String) = (entry._1, entry._2)
-        GrobidScorable.grobidToSlug(json) match {
-          case Some(slug) => new MapFeatures(slug, json)
-          case None => new MapFeatures(Scorable.NoSlug, json)
-        }
-      }
+      .map { entry : (String, String) => GrobidScorable.jsonToMapFeatures(entry._1, entry._2) }
   }
 }
 
@@ -36,14 +31,18 @@ object GrobidScorable {
     HBaseBuilder.build(table, host, List("grobid0:tei_json"), SourceMode.SCAN_ALL)
   }
 
-  def grobidToSlug(json : String) : Option[String] = {
+  def jsonToMapFeatures(key : String, json : String) : MapFeatures = {
     Scorable.jsonToMap(json) match {
-      case None => None
+      case None => MapFeatures(Scorable.NoSlug, json)
       case Some(map) => {
         if (map contains "title") {
-          Some(Scorable.titleToSlug(map("title").asInstanceOf[String]))
+          val map2 = Scorable.toScorableMap(Scorable.getString(map, "title"),
+            sha1=key)
+          new MapFeatures(
+            Scorable.mapToSlug(map2),
+            JSONObject(map2).toString)
         } else {
-          None
+          MapFeatures(Scorable.NoSlug, json)
         }
       }
     }
diff --git a/scalding/src/main/scala/sandcrawler/Scorable.scala b/scalding/src/main/scala/sandcrawler/Scorable.scala
index 0ec8e46..9c8da69 100644
--- a/scalding/src/main/scala/sandcrawler/Scorable.scala
+++ b/scalding/src/main/scala/sandcrawler/Scorable.scala
@@ -2,6 +2,7 @@ package sandcrawler
 
 import scala.math
 import scala.util.parsing.json.JSON
+import scala.util.parsing.json.JSONObject
 
 import cascading.flow.FlowDef
 import com.twitter.scalding._
@@ -36,6 +37,21 @@ object Scorable {
     slug != NoSlug
   }
 
+  // NOTE: I could go all out and make ScorableMap a type.
+  // TODO: Require year. Other features will get added here.
+  def toScorableMap(title : String, year : Int = 0, doi : String = "", sha1 : String = "") : Map[String, Any] = {
+   Map("title" -> title, "year" -> year, "doi" -> doi, "sha1" -> sha1)
+  }
+
+  def toScorableJson(title : String, year : Int, doi : String = "", sha1 : String = "") : String = {
+    JSONObject(toScorableMap(title=title, year=year, doi=doi, sha1=sha1)).toString
+  }
+
+  // TODO: Score on more fields than "title".
+  def isScorableMap(map : Map[String, Any]) : Boolean = {
+    map.contains("title")
+  }
+
   def jsonToMap(json : String) : Option[Map[String, Any]] = {
     // https://stackoverflow.com/a/32717262/631051
     val jsonObject = JSON.parseFull(json)
@@ -46,18 +62,17 @@ object Scorable {
     }
   }
 
-  def titleToSlug(title : String) : String = {
-    if (title == null || title.isEmpty) {
+  // Map should have been produced by toScorableMap.
+  // This guarantees it will have all of the fields needed to compute
+  // the ultimate score, which are a superset of those needed for a slug.
+  def mapToSlug(map : Map[String, Any]) : String = {
+    val unaccented = StringUtilities.removeAccents(getString(map, "title"))
+    // Remove punctuation after splitting on colon.
+    val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase()))
+    if (slug.isEmpty || slug == null) {
       NoSlug
     } else {
-      val unaccented = StringUtilities.removeAccents(title)
-      // Remove punctuation after splitting on colon.
-      val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase()))
-      if (slug.isEmpty || slug == null) {
-        NoSlug
-      } else {
-        slug
-      }
+      slug
     }
   }
 
@@ -68,8 +83,9 @@ object Scorable {
     }
   }
 
-  // Caller is responsible for ensuring that key is in map.
-  def getString(map : Map[String, String], key : String) : String = {
+  // Caller is responsible for ensuring that key is a String in map.
+  // TODO: Add and handle ClassCastException
+  def getString(map : Map[String, Any], key : String) : String = {
     assert(map contains key)
     map(key).asInstanceOf[String]
   }
diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
index 67a8bfe..1c35d66 100644
--- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
@@ -66,20 +66,24 @@ class CrossrefScorableTest extends FlatSpec with Matchers {
   val MalformedCrossrefString = CrossrefString.replace("}", "")
 
   // Unit tests
-/*
-  "crossrefToSlug()" should "get the right slug for a crossref json string" in {
-    val slug = CrossrefScorable.crossrefToSlug(CrossrefStringWithTitle)
-    slug should contain ("sometitle")
+  "simplifyJson()" should "return None for bad JSON" in {
+    CrossrefScorable.simplifyJson("") shouldBe None
+    CrossrefScorable.simplifyJson(MalformedCrossrefString) shouldBe None
   }
 
-  it should "return None if given json string without title" in {
-    val slug = CrossrefScorable.crossrefToSlug(CrossrefStringWithoutTitle)
-    slug shouldBe None
+  it should "return None for JSON lacking title" in {
+    CrossrefScorable.simplifyJson(CrossrefStringWithoutTitle) shouldBe None
   }
 
-  it should "return None if given a malformed json string" in {
-    val slug = CrossrefScorable.crossrefToSlug(MalformedCrossrefString)
-    slug shouldBe None
+  it should "return appropriate result for valid JSON" in {
+    CrossrefScorable.simplifyJson(CrossrefStringWithTitle) match {
+      case None => fail("None unexpectedly returned by simplifyJson")
+      case Some(map) => {
+        Scorable.isScorableMap(map) shouldBe true
+        map.size shouldBe 1
+        map.keys should contain ("title")
+        map("title") shouldBe "SomeTitle"
+      }
+    }
   }
- */
 }
diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
index 7777610..5bb955a 100644
--- a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -60,18 +60,15 @@ class GrobidScorableTest extends FlatSpec with Matchers {
 
   // Unit tests
 
-  "grobidToSlug()" should "get the right slug for a grobid json string" in {
-    val slug = GrobidScorable.grobidToSlug(GrobidStringWithTitle)
-    slug should contain ("dummy example file")
+  "GrobidScorable.jsonToMapFeatures()" should "handle invalid JSON" in {
+    val result = GrobidScorable.jsonToMapFeatures(MalformedGrobidString) shouldBe None
+    result.slug shouldBe Scorable.NoSlug
+    result.json shouldBe MalformedGrobidString
   }
 
-  it should "return None if given json string without title" in {
-    val slug = GrobidScorable.grobidToSlug(GrobidStringWithoutTitle)
-    slug shouldBe None
-  }
-
-  it should "return None if given a malformed json string" in {
-    val slug = GrobidScorable.grobidToSlug(MalformedGrobidString)
-    slug shouldBe None
+  "GrobidScorable.jsonToMapFeatures()" should "handle missing title" in {
+    val result = GrobidScorable.jsonToMapFeatures(GrobidStringWithoutTitle) shouldBe None
+    result.slug shouldBe Scorable.NoSlug
+    result.json shouldBe GrobidStringWithoutTitle
   }
 }
-- 
cgit v1.2.3


From 31354b1a6062c5c56a30610f68fa48c82a7e83f0 Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Sun, 12 Aug 2018 18:08:51 -0700
Subject: Tests pass.

---
 scalding/src/main/scala/sandcrawler/Scorable.scala | 11 +--
 .../scala/sandcrawler/CrossrefScorableTest.scala   | 89 ----------------------
 .../scala/sandcrawler/GrobidScorableTest.scala     | 20 +++--
 .../src/test/scala/sandcrawler/ScorableTest.scala  | 28 ++++---
 4 files changed, 39 insertions(+), 109 deletions(-)
 delete mode 100644 scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala

(limited to 'scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala')

diff --git a/scalding/src/main/scala/sandcrawler/Scorable.scala b/scalding/src/main/scala/sandcrawler/Scorable.scala
index 9c8da69..929461b 100644
--- a/scalding/src/main/scala/sandcrawler/Scorable.scala
+++ b/scalding/src/main/scala/sandcrawler/Scorable.scala
@@ -66,13 +66,14 @@ object Scorable {
   // This guarantees it will have all of the fields needed to compute
   // the ultimate score, which are a superset of those needed for a slug.
   def mapToSlug(map : Map[String, Any]) : String = {
-    val unaccented = StringUtilities.removeAccents(getString(map, "title"))
-    // Remove punctuation after splitting on colon.
-    val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase()))
-    if (slug.isEmpty || slug == null) {
+    val title = getString(map, "title")
+    if (title == null) {
       NoSlug
     } else {
-      slug
+      val unaccented = StringUtilities.removeAccents(title)
+      // Remove punctuation after splitting on colon.
+      val slug = StringUtilities.removePunctuation((unaccented.split(":")(0).toLowerCase())).replaceAll("\\s", "")
+      if (slug.isEmpty || slug == null) NoSlug else slug
     }
   }
 
diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
deleted file mode 100644
index 1c35d66..0000000
--- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
+++ /dev/null
@@ -1,89 +0,0 @@
-package sandcrawler
-
-import cascading.tuple.Fields
-import cascading.tuple.Tuple
-import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
-import org.apache.hadoop.hbase.io.ImmutableBytesWritable
-import org.apache.hadoop.hbase.util.Bytes
-import org.scalatest._
-import parallelai.spyglass.hbase.HBaseConstants.SourceMode
-
-class CrossrefScorableTest extends FlatSpec with Matchers {
-  val CrossrefString =
-"""
-{ "_id" : { "$oid" : "5a553d5988a035a45bf50ed3" }, 
-  "indexed" : { "date-parts" : [ [ 2017, 10, 23 ] ], 
-    "date-time" : "2017-10-23T17:19:16Z", 
-    "timestamp" : { "$numberLong" : "1508779156477" } }, 
-  "reference-count" : 0, 
-  "publisher" : "Elsevier BV", 
-  "issue" : "3", 
-  "license" : [ { "URL" : "http://www.elsevier.com/tdm/userlicense/1.0/", 
-                    "start" : { "date-parts" : [ [ 1996, 1, 1 ] ], 
-                                "date-time" : "1996-01-01T00:00:00Z", 
-                                "timestamp" : { "$numberLong" : "820454400000" } }, 
-                                "delay-in-days" : 0, "content-version" : "tdm" }],
-  "content-domain" : { "domain" : [], "crossmark-restriction" : false }, 
-  "published-print" : { "date-parts" : [ [ 1996 ] ] }, 
-  "DOI" : "<<DOI>>",
-  "type" : "journal-article", 
-  "created" : { "date-parts" : [ [ 2002, 7, 25 ] ], 
-    "date-time" : "2002-07-25T15:09:41Z", 
-    "timestamp" : { "$numberLong" : "1027609781000" } }, 
-  "page" : "186-187", 
-  "source" : "Crossref", 
-  "is-referenced-by-count" : 0, 
-  "title" : [ "<<TITLE>>" ],
-  "prefix" : "10.1016", 
-  "volume" : "9", 
-  "author" : [ { "given" : "W", "family" : "Gaier", "affiliation" : [] } ], 
-  "member" : "78", 
-  "container-title" : [ "Journal de Pédiatrie et de Puériculture" ], 
-  "link" : [ { "URL" :  "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/xml",
-               "content-type" : "text/xml", 
-                 "content-version" : "vor",
-                 "intended-application" : "text-mining" }, 
-               { "URL" :
-  "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/plain",
-                 "content-type" : "text/plain", 
-                 "content-version" : "vor",
-                 "intended-application" : "text-mining" } ], 
-  "deposited" : { "date-parts" : [ [ 2015, 9, 3 ] ], 
-                  "date-time" : "2015-09-03T10:03:43Z", 
-                  "timestamp" : { "$numberLong" : "1441274623000" } }, 
-  "score" : 1, 
-  "issued" : { "date-parts" : [ [ 1996 ] ] }, 
-  "references-count" : 0, 
-  "alternative-id" : [ "0987-7983(96)87729-2" ], 
-  "URL" : "http://dx.doi.org/10.1016/0987-7983(96)87729-2", 
-  "ISSN" : [ "0987-7983" ], 
-  "issn-type" : [ { "value" : "0987-7983", "type" : "print" } ], 
-  "subject" : [ "Pediatrics, Perinatology, and Child Health" ]
-}
-"""
-  val CrossrefStringWithTitle = CrossrefString.replace("<<TITLE>>", "SomeTitle")
-  val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle")
-  val MalformedCrossrefString = CrossrefString.replace("}", "")
-
-  // Unit tests
-  "simplifyJson()" should "return None for bad JSON" in {
-    CrossrefScorable.simplifyJson("") shouldBe None
-    CrossrefScorable.simplifyJson(MalformedCrossrefString) shouldBe None
-  }
-
-  it should "return None for JSON lacking title" in {
-    CrossrefScorable.simplifyJson(CrossrefStringWithoutTitle) shouldBe None
-  }
-
-  it should "return appropriate result for valid JSON" in {
-    CrossrefScorable.simplifyJson(CrossrefStringWithTitle) match {
-      case None => fail("None unexpectedly returned by simplifyJson")
-      case Some(map) => {
-        Scorable.isScorableMap(map) shouldBe true
-        map.size shouldBe 1
-        map.keys should contain ("title")
-        map("title") shouldBe "SomeTitle"
-      }
-    }
-  }
-}
diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
index 5bb955a..3fcd856 100644
--- a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -57,18 +57,28 @@ class GrobidScorableTest extends FlatSpec with Matchers {
   val GrobidStringWithTitle = GrobidString.replace("<<TITLE>>", "Dummy Example File")
   val GrobidStringWithoutTitle = GrobidString.replace("title", "nottitle")
   val MalformedGrobidString = GrobidString.replace("}", "")
+  val Key = "Dummy Key"
 
   // Unit tests
 
   "GrobidScorable.jsonToMapFeatures()" should "handle invalid JSON" in {
-    val result = GrobidScorable.jsonToMapFeatures(MalformedGrobidString) shouldBe None
+    val result = GrobidScorable.jsonToMapFeatures(Key, MalformedGrobidString) 
     result.slug shouldBe Scorable.NoSlug
-    result.json shouldBe MalformedGrobidString
   }
 
-  "GrobidScorable.jsonToMapFeatures()" should "handle missing title" in {
-    val result = GrobidScorable.jsonToMapFeatures(GrobidStringWithoutTitle) shouldBe None
+  it should "handle missing title" in {
+    val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithoutTitle)
     result.slug shouldBe Scorable.NoSlug
-    result.json shouldBe GrobidStringWithoutTitle
+  }
+
+  it should "handle valid input" in {
+    val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithTitle)
+    result.slug shouldBe "dummyexamplefile"
+    Scorable.jsonToMap(result.json) match {
+      case None => fail()
+      case Some(map) => {
+        map("title").asInstanceOf[String] shouldBe "Dummy Example File"
+      }
+    }
   }
 }
diff --git a/scalding/src/test/scala/sandcrawler/ScorableTest.scala b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
index 2f80492..95faacc 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
@@ -54,28 +54,36 @@ class ScorableTest extends FlatSpec with Matchers {
   "annex": null
 }
 """
+  private def titleToSlug(s : String) : String = {
+    Scorable.mapToSlug(Scorable.toScorableMap(title = s))
+  }
 
-  "titleToSlug()" should "extract the parts of titles before a colon" in {
-    Scorable.titleToSlug("HELLO:there") shouldBe "hello"
+  "mapToSlug()" should "extract the parts of titles before a colon" in {
+    titleToSlug("HELLO:there") shouldBe "hello"
   }
 
   it should "extract an entire colon-less string" in {
-    Scorable.titleToSlug("hello THERE") shouldBe "hello there"
+    titleToSlug("hello THERE") shouldBe "hellothere"
   }
 
   it should "return Scorable.NoSlug if given empty string" in {
-    Scorable.titleToSlug("") shouldBe Scorable.NoSlug
+    titleToSlug("") shouldBe Scorable.NoSlug
   }
 
   it should "return Scorable.NoSlug if given null" in {
-    Scorable.titleToSlug(null) shouldBe Scorable.NoSlug
+    titleToSlug(null) shouldBe Scorable.NoSlug
+  }
+
+  it should "strip punctuation" in {
+    titleToSlug("HELLO!:the:re") shouldBe "hello"
+    titleToSlug("a:b:c") shouldBe "a"
+    titleToSlug(
+      "If you're happy and you know it, clap your hands!") shouldBe "ifyourehappyandyouknowitclapyourhands"
   }
 
-  "titleToSlug()" should "strip punctuation" in {
-    Scorable.titleToSlug("HELLO!:the:re") shouldBe "hello"
-    Scorable.titleToSlug("a:b:c") shouldBe "a"
-    Scorable.titleToSlug(
-      "If you're happy and you know it, clap your hands!") shouldBe "if youre happy and you know it clap your hands"
+  it should "remove whitespace" in {
+    titleToSlug("foo bar : baz ::") shouldBe "foobar"
+    titleToSlug("\na\t:b:c") shouldBe "a"
   }
 
   "jsonToMap()" should "return a map, given a legal JSON string" in {
-- 
cgit v1.2.3


From 5615428921a45ba6a2fb005b255a28dcbb83b13f Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Sun, 12 Aug 2018 19:12:32 -0700
Subject: Snapshot before changing Scorable to find bug.

---
 .../main/scala/sandcrawler/CrossrefScorable.scala  | 41 ++++++++++++----------
 scalding/src/main/scala/sandcrawler/Scorable.scala |  1 -
 .../scala/sandcrawler/CrossrefScorableTest.scala   | 24 ++++++-------
 .../scala/sandcrawler/GrobidScorableTest.scala     |  1 +
 .../src/test/scala/sandcrawler/ScoreJobTest.scala  | 15 +++++---
 5 files changed, 46 insertions(+), 36 deletions(-)

(limited to 'scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala')

diff --git a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
index 667a5cc..e257152 100644
--- a/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
+++ b/scalding/src/main/scala/sandcrawler/CrossrefScorable.scala
@@ -41,26 +41,31 @@ class CrossrefScorable extends Scorable with HBasePipeConversions {
   def getFeaturesPipe(args : Args)(implicit mode : Mode, flowDef : FlowDef) : TypedPipe[MapFeatures] = {
     getSource(args).read
       .toTypedPipe[String](new Fields("line"))
-      .map{ json : String =>
-        Scorable.jsonToMap(json) match {
-          case None => MapFeatures(Scorable.NoSlug, json)
-          case Some(map) => {
-            if ((map contains "title") && (map contains "DOI")) {
-              val titles = map("title").asInstanceOf[List[String]]
-              if (titles.isEmpty) {
-                new MapFeatures(Scorable.NoSlug, json)
-              } else {
-                val title = titles(0)
-                val map2 = Scorable.toScorableMap(title=titles(0), doi=map("DOI").asInstanceOf[String])
-                new MapFeatures(
-                  Scorable.mapToSlug(map2),
-                  JSONObject(map2).toString)
-              }
-            } else {
-              new MapFeatures(Scorable.NoSlug, json)
-            }
+      .map { CrossrefScorable.jsonToMapFeatures(_) }
+  }
+}
+
+object CrossrefScorable {
+  def jsonToMapFeatures(json : String) : MapFeatures = {
+    Scorable.jsonToMap(json) match {
+      case None => MapFeatures(Scorable.NoSlug, json)
+      case Some(map) => {
+        if ((map contains "titles") && (map contains "DOI")) {
+          val titles = map("titles").asInstanceOf[List[String]]
+          val doi = Scorable.getString(map, "DOI")
+          if (titles.isEmpty || titles == null || doi.isEmpty || doi == null) {
+            new MapFeatures(Scorable.NoSlug, json)
+          } else {
+            val title = titles(0)
+            val map2 = Scorable.toScorableMap(title=title, doi=doi)
+            new MapFeatures(
+              Scorable.mapToSlug(map2),
+              JSONObject(map2).toString)
           }
+        } else {
+          new MapFeatures(Scorable.NoSlug, json)
         }
       }
+    }
   }
 }
diff --git a/scalding/src/main/scala/sandcrawler/Scorable.scala b/scalding/src/main/scala/sandcrawler/Scorable.scala
index 929461b..a256fa4 100644
--- a/scalding/src/main/scala/sandcrawler/Scorable.scala
+++ b/scalding/src/main/scala/sandcrawler/Scorable.scala
@@ -7,7 +7,6 @@ import scala.util.parsing.json.JSONObject
 import cascading.flow.FlowDef
 import com.twitter.scalding._
 import com.twitter.scalding.typed.TDsl._
-//import TDsl._
 
 case class MapFeatures(slug : String, json : String)
 case class ReduceFeatures(json : String)
diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
index 1c35d66..dc6f347 100644
--- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
@@ -66,23 +66,23 @@ class CrossrefScorableTest extends FlatSpec with Matchers {
   val MalformedCrossrefString = CrossrefString.replace("}", "")
 
   // Unit tests
-  "simplifyJson()" should "return None for bad JSON" in {
-    CrossrefScorable.simplifyJson("") shouldBe None
-    CrossrefScorable.simplifyJson(MalformedCrossrefString) shouldBe None
+  "CrossrefScorable.jsonToMapFeatures()" should "handle invalid JSON" in {
+    val result = CrossrefScorable.jsonToMapFeatures(MalformedCrossrefString) 
+    result.slug shouldBe Scorable.NoSlug
   }
 
-  it should "return None for JSON lacking title" in {
-    CrossrefScorable.simplifyJson(CrossrefStringWithoutTitle) shouldBe None
+  it should "handle missing title" in {
+    val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithoutTitle)
+    result.slug shouldBe Scorable.NoSlug
   }
 
-  it should "return appropriate result for valid JSON" in {
-    CrossrefScorable.simplifyJson(CrossrefStringWithTitle) match {
-      case None => fail("None unexpectedly returned by simplifyJson")
+  it should "handle valid input" in {
+    val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithTitle)
+    result.slug shouldBe "dummyexamplefile"
+    Scorable.jsonToMap(result.json) match {
+      case None => fail()
       case Some(map) => {
-        Scorable.isScorableMap(map) shouldBe true
-        map.size shouldBe 1
-        map.keys should contain ("title")
-        map("title") shouldBe "SomeTitle"
+        map("title").asInstanceOf[String] shouldBe "Dummy Example File"
       }
     }
   }
diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
index 3fcd856..4b958b9 100644
--- a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -77,6 +77,7 @@ class GrobidScorableTest extends FlatSpec with Matchers {
     Scorable.jsonToMap(result.json) match {
       case None => fail()
       case Some(map) => {
+        map should contain key "title"
         map("title").asInstanceOf[String] shouldBe "Dummy Example File"
       }
     }
diff --git a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
index 8acb454..8436817 100644
--- a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
@@ -149,11 +149,16 @@ class ScoreJobTest extends FlatSpec with Matchers {
       2 -> CrossrefString.replace("<<TITLE>>", "Title 1: TNG 3").replace("<<DOI>>", "DOI-0.75"),
       3 -> CrossrefString.replace("<<TITLE>>", "Title 2: Rebooted").replace("<<DOI>>", "DOI-1")))
     .sink[(String, Int, String, String)](TypedTsv[(String, Int, String, String)](output)) {
-      // Grobid titles: 
-      //   "Title 1", "Title 2: TNG", "Title 3: The Sequel"
-      // crossref slugs: 
-      //   "Title 1: TNG", "Title 1: TNG 2", "Title 1: TNG 3", "Title 2 Rebooted"
-      // Join should have 3 "Title  1" slugs and 1 "Title 2" slug
+      // Grobid titles and slugs (in parentheses): 
+      //   Title 1                       (title1)
+      //   Title 2: TNG                  (title2)
+      //   Title 3: The Sequel           (title3)
+      // crossref titles and slugs (in parentheses):
+      //   Title 1: TNG                  (title1)
+      //   Title 1: TNG 2                (title1)
+      //   Title 1: TNG 3                (title1)
+      //   Title 2 Rebooted              (title2rebooted)
+      // Join should have 3 "title1" slugs and 1 "title2" slug
       outputBuffer => 
       "The pipeline" should "return a 4-element list" in {
         outputBuffer should have length 4
-- 
cgit v1.2.3


From 3ff30c8f20d36f8e47ec5478c10c3348d2f45fa6 Mon Sep 17 00:00:00 2001
From: Ellen Spertus <ellen.spertus@gmail.com>
Date: Tue, 14 Aug 2018 20:38:29 -0700
Subject: Fixed style problems (or disabled warning when appropriate) for
 tests.

---
 scalding/build.sbt                                 |  7 ++
 .../scala/sandcrawler/CrossrefScorableTest.scala   | 87 ++++++++++---------
 .../scala/sandcrawler/GrobidScorableTest.scala     |  7 +-
 .../test/scala/sandcrawler/HBaseBuilderTest.scala  |  1 +
 .../scala/sandcrawler/HBaseMimeCountTest.scala     |  9 +-
 .../test/scala/sandcrawler/HBaseRowCountTest.scala | 11 +--
 .../scala/sandcrawler/HBaseStatusCountTest.scala   | 10 ++-
 .../scala/sandcrawler/ScorableFeaturesTest.scala   |  1 +
 .../src/test/scala/sandcrawler/ScorableTest.scala  |  5 +-
 .../src/test/scala/sandcrawler/ScoreJobTest.scala  | 97 ++++++++++++----------
 10 files changed, 135 insertions(+), 100 deletions(-)

(limited to 'scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala')

diff --git a/scalding/build.sbt b/scalding/build.sbt
index 2addd60..d477399 100644
--- a/scalding/build.sbt
+++ b/scalding/build.sbt
@@ -20,6 +20,13 @@ lazy val root = (project in file(".")).
       scalaSourceFiles.filterNot(_.getAbsolutePath.contains(dirNameToExclude))
     },
 
+    (scalastyleSources in Test) := {
+      // all .scala files in "src/test/scala"
+      val scalaSourceFiles = ((scalaSource in Test).value ** "*.scala").get    
+      val dirNameToExclude = "/example/"
+      scalaSourceFiles.filterNot(_.getAbsolutePath.contains(dirNameToExclude))
+    },
+
     name := "sandcrawler",
 
     resolvers += "conjars.org" at "http://conjars.org/repo",
diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
index 75be03e..e171dba 100644
--- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
@@ -2,72 +2,77 @@ package sandcrawler
 
 import cascading.tuple.Fields
 import cascading.tuple.Tuple
-import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.TextLine
+import com.twitter.scalding.TupleConversions
+import com.twitter.scalding.TypedTsv
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.scalatest._
 import parallelai.spyglass.hbase.HBaseConstants.SourceMode
 
 class CrossrefScorableTest extends FlatSpec with Matchers {
+  // scalastyle:off
   val CrossrefString =
 """
-{ "_id" : { "$oid" : "5a553d5988a035a45bf50ed3" }, 
-  "indexed" : { "date-parts" : [ [ 2017, 10, 23 ] ], 
-    "date-time" : "2017-10-23T17:19:16Z", 
-    "timestamp" : { "$numberLong" : "1508779156477" } }, 
-  "reference-count" : 0, 
-  "publisher" : "Elsevier BV", 
-  "issue" : "3", 
-  "license" : [ { "URL" : "http://www.elsevier.com/tdm/userlicense/1.0/", 
-                    "start" : { "date-parts" : [ [ 1996, 1, 1 ] ], 
-                                "date-time" : "1996-01-01T00:00:00Z", 
-                                "timestamp" : { "$numberLong" : "820454400000" } }, 
+{ "_id" : { "$oid" : "5a553d5988a035a45bf50ed3" },
+  "indexed" : { "date-parts" : [ [ 2017, 10, 23 ] ],
+    "date-time" : "2017-10-23T17:19:16Z",
+    "timestamp" : { "$numberLong" : "1508779156477" } },
+  "reference-count" : 0,
+  "publisher" : "Elsevier BV",
+  "issue" : "3",
+  "license" : [ { "URL" : "http://www.elsevier.com/tdm/userlicense/1.0/",
+                    "start" : { "date-parts" : [ [ 1996, 1, 1 ] ],
+                                "date-time" : "1996-01-01T00:00:00Z",
+                                "timestamp" : { "$numberLong" : "820454400000" } },
                                 "delay-in-days" : 0, "content-version" : "tdm" }],
-  "content-domain" : { "domain" : [], "crossmark-restriction" : false }, 
-  "published-print" : { "date-parts" : [ [ 1996 ] ] }, 
+  "content-domain" : { "domain" : [], "crossmark-restriction" : false },
+  "published-print" : { "date-parts" : [ [ 1996 ] ] },
   "DOI" : "<<DOI>>",
-  "type" : "journal-article", 
-  "created" : { "date-parts" : [ [ 2002, 7, 25 ] ], 
-    "date-time" : "2002-07-25T15:09:41Z", 
-    "timestamp" : { "$numberLong" : "1027609781000" } }, 
-  "page" : "186-187", 
-  "source" : "Crossref", 
-  "is-referenced-by-count" : 0, 
+  "type" : "journal-article",
+  "created" : { "date-parts" : [ [ 2002, 7, 25 ] ],
+    "date-time" : "2002-07-25T15:09:41Z",
+    "timestamp" : { "$numberLong" : "1027609781000" } },
+  "page" : "186-187",
+  "source" : "Crossref",
+  "is-referenced-by-count" : 0,
   "title" : [ "<<TITLE>>" ],
-  "prefix" : "10.1016", 
-  "volume" : "9", 
-  "author" : [ { "given" : "W", "family" : "Gaier", "affiliation" : [] } ], 
-  "member" : "78", 
-  "container-title" : [ "Journal de Pédiatrie et de Puériculture" ], 
+  "prefix" : "10.1016",
+  "volume" : "9",
+  "author" : [ { "given" : "W", "family" : "Gaier", "affiliation" : [] } ],
+  "member" : "78",
+  "container-title" : [ "Journal de Pédiatrie et de Puériculture" ],
   "link" : [ { "URL" :  "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/xml",
-               "content-type" : "text/xml", 
+               "content-type" : "text/xml",
                  "content-version" : "vor",
-                 "intended-application" : "text-mining" }, 
+                 "intended-application" : "text-mining" },
                { "URL" :
   "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/plain",
-                 "content-type" : "text/plain", 
+                 "content-type" : "text/plain",
                  "content-version" : "vor",
-                 "intended-application" : "text-mining" } ], 
-  "deposited" : { "date-parts" : [ [ 2015, 9, 3 ] ], 
-                  "date-time" : "2015-09-03T10:03:43Z", 
-                  "timestamp" : { "$numberLong" : "1441274623000" } }, 
-  "score" : 1, 
-  "issued" : { "date-parts" : [ [ 1996 ] ] }, 
-  "references-count" : 0, 
-  "alternative-id" : [ "0987-7983(96)87729-2" ], 
-  "URL" : "http://dx.doi.org/10.1016/0987-7983(96)87729-2", 
-  "ISSN" : [ "0987-7983" ], 
-  "issn-type" : [ { "value" : "0987-7983", "type" : "print" } ], 
+                 "intended-application" : "text-mining" } ],
+  "deposited" : { "date-parts" : [ [ 2015, 9, 3 ] ],
+                  "date-time" : "2015-09-03T10:03:43Z",
+                  "timestamp" : { "$numberLong" : "1441274623000" } },
+  "score" : 1,
+  "issued" : { "date-parts" : [ [ 1996 ] ] },
+  "references-count" : 0,
+  "alternative-id" : [ "0987-7983(96)87729-2" ],
+  "URL" : "http://dx.doi.org/10.1016/0987-7983(96)87729-2",
+  "ISSN" : [ "0987-7983" ],
+  "issn-type" : [ { "value" : "0987-7983", "type" : "print" } ],
   "subject" : [ "Pediatrics, Perinatology, and Child Health" ]
 }
 """
+  // scalastyle:on
   val CrossrefStringWithTitle = CrossrefString.replace("<<TITLE>>", "Some Title")
   val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle")
   val MalformedCrossrefString = CrossrefString.replace("}", "")
 
   // Unit tests
   "CrossrefScorable.jsonToMapFeatures()" should "handle invalid JSON" in {
-    val result = CrossrefScorable.jsonToMapFeatures(MalformedCrossrefString) 
+    val result = CrossrefScorable.jsonToMapFeatures(MalformedCrossrefString)
     result.slug shouldBe Scorable.NoSlug
   }
 
diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
index 4b958b9..661824b 100644
--- a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -2,7 +2,10 @@ package sandcrawler
 
 import cascading.tuple.Fields
 import cascading.tuple.Tuple
-import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.TextLine
+import com.twitter.scalding.TupleConversions
+import com.twitter.scalding.TypedTsv
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.scalatest._
@@ -62,7 +65,7 @@ class GrobidScorableTest extends FlatSpec with Matchers {
   // Unit tests
 
   "GrobidScorable.jsonToMapFeatures()" should "handle invalid JSON" in {
-    val result = GrobidScorable.jsonToMapFeatures(Key, MalformedGrobidString) 
+    val result = GrobidScorable.jsonToMapFeatures(Key, MalformedGrobidString)
     result.slug shouldBe Scorable.NoSlug
   }
 
diff --git a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
index 603a4c7..c61cb22 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseBuilderTest.scala
@@ -22,6 +22,7 @@ class HBaseBuilderTest extends FlatSpec with Matchers {
     fields should have length 0
   }
 
+  //scalastyle:off no.whitespace.before.left.bracket
   it should "throw IllegalArgumentException on malformed input" in {
     a [IllegalArgumentException] should be thrownBy {
       HBaseBuilder.parseColSpecs(List("file_size"))
diff --git a/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
index fde2290..d6d283f 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseMimeCountTest.scala
@@ -1,15 +1,18 @@
 package sandcrawler
 
-import cascading.tuple.{Tuple, Fields}
-import com.twitter.scalding.{JobTest, Tsv, TupleConversions}
+import cascading.tuple.Fields
+import cascading.tuple.Tuple
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.Tsv
+import com.twitter.scalding.TupleConversions
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.junit.runner.RunWith
 import org.scalatest.FunSpec
 import org.scalatest.junit.JUnitRunner
 import org.slf4j.LoggerFactory
-import parallelai.spyglass.hbase.HBaseSource
 import parallelai.spyglass.hbase.HBaseConstants.SourceMode
+import parallelai.spyglass.hbase.HBaseSource
 import scala._
 
 @RunWith(classOf[JUnitRunner])
diff --git a/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala
index 3424a36..c4ca5aa 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseRowCountTest.scala
@@ -1,15 +1,18 @@
 package sandcrawler
 
-import cascading.tuple.{Tuple, Fields}
-import com.twitter.scalding.{JobTest, Tsv, TupleConversions}
+import cascading.tuple.Fields
+import cascading.tuple.Tuple
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.Tsv
+import com.twitter.scalding.TupleConversions
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.junit.runner.RunWith
 import org.scalatest.FunSpec
 import org.scalatest.junit.JUnitRunner
 import org.slf4j.LoggerFactory
-import parallelai.spyglass.hbase.HBaseSource
 import parallelai.spyglass.hbase.HBaseConstants.SourceMode
+import parallelai.spyglass.hbase.HBaseSource
 import scala._
 
 /**
@@ -47,12 +50,10 @@ class HBaseRowCountTest extends FunSpec with TupleConversions {
       outputBuffer =>
 
         it("should return the test data provided.") {
-          println("outputBuffer.size => " + outputBuffer.size)
           assert(outputBuffer.size === 1)
         }
 
         it("should return the correct count") {
-          println("raw output => " + outputBuffer)
           assert(outputBuffer(0).getObject(0) === 8)
         }
     }
diff --git a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
index 8a71f31..fe3ff21 100644
--- a/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
+++ b/scalding/src/test/scala/sandcrawler/HBaseStatusCountTest.scala
@@ -1,15 +1,19 @@
 package sandcrawler
 
-import cascading.tuple.{Tuple, Fields}
-import com.twitter.scalding.{JobTest, Tsv, TypedTsv, TupleConversions}
+import cascading.tuple.Fields
+import cascading.tuple.Tuple
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.Tsv
+import com.twitter.scalding.TupleConversions
+import com.twitter.scalding.TypedTsv
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.junit.runner.RunWith
 import org.scalatest.FunSpec
 import org.scalatest.junit.JUnitRunner
 import org.slf4j.LoggerFactory
-import parallelai.spyglass.hbase.HBaseSource
 import parallelai.spyglass.hbase.HBaseConstants.SourceMode
+import parallelai.spyglass.hbase.HBaseSource
 import scala._
 
 @RunWith(classOf[JUnitRunner])
diff --git a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
index 7ec0c4d..f9c30a2 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
@@ -2,6 +2,7 @@ package sandcrawler
 
 import org.scalatest._
 
+// scalastyle:off null
 class ScorableFeaturesTest extends FlatSpec with Matchers {
   private def titleToSlug(s : String) : String = {
     new ScorableFeatures(title = s).toSlug
diff --git a/scalding/src/test/scala/sandcrawler/ScorableTest.scala b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
index fd44f57..f63bef8 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
@@ -2,7 +2,10 @@ package sandcrawler
 
 import cascading.tuple.Fields
 import cascading.tuple.Tuple
-import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.TextLine
+import com.twitter.scalding.TupleConversions
+import com.twitter.scalding.TypedTsv
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.scalatest._
diff --git a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
index 1c6ae83..34081a5 100644
--- a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
@@ -2,13 +2,17 @@ package sandcrawler
 
 import cascading.tuple.Fields
 import cascading.tuple.Tuple
-import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import com.twitter.scalding.JobTest
+import com.twitter.scalding.TextLine
+import com.twitter.scalding.TupleConversions
+import com.twitter.scalding.TypedTsv
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable
 import org.apache.hadoop.hbase.util.Bytes
 import org.scalatest._
 import parallelai.spyglass.hbase.HBaseConstants.SourceMode
 
 class ScoreJobTest extends FlatSpec with Matchers {
+  //scalastyle:off
   val JsonString = """
 {
   "title": "<<TITLE>>",
@@ -54,62 +58,65 @@ class ScoreJobTest extends FlatSpec with Matchers {
   "annex": null
 }
 """
+  // scalastyle:on
   val JsonStringWithTitle = JsonString.replace("<<TITLE>>", "Dummy Example File")
   val JsonStringWithoutTitle = JsonString.replace("title", "nottitle")
   val MalformedJsonString = JsonString.replace("}", "")
 
+  // scalastyle:off
   val CrossrefString =
 """
-{ "_id" : { "$oid" : "5a553d5988a035a45bf50ed3" }, 
-  "indexed" : { "date-parts" : [ [ 2017, 10, 23 ] ], 
-    "date-time" : "2017-10-23T17:19:16Z", 
-    "timestamp" : { "$numberLong" : "1508779156477" } }, 
-  "reference-count" : 0, 
-  "publisher" : "Elsevier BV", 
-  "issue" : "3", 
-  "license" : [ { "URL" : "http://www.elsevier.com/tdm/userlicense/1.0/", 
-                    "start" : { "date-parts" : [ [ 1996, 1, 1 ] ], 
-                                "date-time" : "1996-01-01T00:00:00Z", 
-                                "timestamp" : { "$numberLong" : "820454400000" } }, 
+{ "_id" : { "$oid" : "5a553d5988a035a45bf50ed3" },
+  "indexed" : { "date-parts" : [ [ 2017, 10, 23 ] ],
+    "date-time" : "2017-10-23T17:19:16Z",
+    "timestamp" : { "$numberLong" : "1508779156477" } },
+  "reference-count" : 0,
+  "publisher" : "Elsevier BV",
+  "issue" : "3",
+  "license" : [ { "URL" : "http://www.elsevier.com/tdm/userlicense/1.0/",
+                    "start" : { "date-parts" : [ [ 1996, 1, 1 ] ],
+                                "date-time" : "1996-01-01T00:00:00Z",
+                                "timestamp" : { "$numberLong" : "820454400000" } },
                                 "delay-in-days" : 0, "content-version" : "tdm" }],
-  "content-domain" : { "domain" : [], "crossmark-restriction" : false }, 
-  "published-print" : { "date-parts" : [ [ 1996 ] ] }, 
+  "content-domain" : { "domain" : [], "crossmark-restriction" : false },
+  "published-print" : { "date-parts" : [ [ 1996 ] ] },
   "DOI" : "<<DOI>>",
-  "type" : "journal-article", 
-  "created" : { "date-parts" : [ [ 2002, 7, 25 ] ], 
-    "date-time" : "2002-07-25T15:09:41Z", 
-    "timestamp" : { "$numberLong" : "1027609781000" } }, 
-  "page" : "186-187", 
-  "source" : "Crossref", 
-  "is-referenced-by-count" : 0, 
+  "type" : "journal-article",
+  "created" : { "date-parts" : [ [ 2002, 7, 25 ] ],
+    "date-time" : "2002-07-25T15:09:41Z",
+    "timestamp" : { "$numberLong" : "1027609781000" } },
+  "page" : "186-187",
+  "source" : "Crossref",
+  "is-referenced-by-count" : 0,
   "title" : [ "<<TITLE>>" ],
-  "prefix" : "10.1016", 
-  "volume" : "9", 
-  "author" : [ { "given" : "W", "family" : "Gaier", "affiliation" : [] } ], 
-  "member" : "78", 
-  "container-title" : [ "Journal de Pédiatrie et de Puériculture" ], 
+  "prefix" : "10.1016",
+  "volume" : "9",
+  "author" : [ { "given" : "W", "family" : "Gaier", "affiliation" : [] } ],
+  "member" : "78",
+  "container-title" : [ "Journal de Pédiatrie et de Puériculture" ],
   "link" : [ { "URL" :  "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/xml",
-               "content-type" : "text/xml", 
-                 "content-version" : "vor",
-                 "intended-application" : "text-mining" }, 
+               "content-type" : "text/xml",
+               "content-version" : "vor",
+               "intended-application" : "text-mining" },
                { "URL" :
   "http://api.elsevier.com/content/article/PII:0987-7983(96)87729-2?httpAccept=text/plain",
-                 "content-type" : "text/plain", 
+                 "content-type" : "text/plain",
                  "content-version" : "vor",
-                 "intended-application" : "text-mining" } ], 
-  "deposited" : { "date-parts" : [ [ 2015, 9, 3 ] ], 
-                  "date-time" : "2015-09-03T10:03:43Z", 
-                  "timestamp" : { "$numberLong" : "1441274623000" } }, 
-  "score" : 1, 
-  "issued" : { "date-parts" : [ [ 1996 ] ] }, 
-  "references-count" : 0, 
-  "alternative-id" : [ "0987-7983(96)87729-2" ], 
-  "URL" : "http://dx.doi.org/10.1016/0987-7983(96)87729-2", 
-  "ISSN" : [ "0987-7983" ], 
-  "issn-type" : [ { "value" : "0987-7983", "type" : "print" } ], 
+                 "intended-application" : "text-mining" } ],
+  "deposited" : { "date-parts" : [ [ 2015, 9, 3 ] ],
+                  "date-time" : "2015-09-03T10:03:43Z",
+                  "timestamp" : { "$numberLong" : "1441274623000" } },
+  "score" : 1,
+  "issued" : { "date-parts" : [ [ 1996 ] ] },
+  "references-count" : 0,
+  "alternative-id" : [ "0987-7983(96)87729-2" ],
+  "URL" : "http://dx.doi.org/10.1016/0987-7983(96)87729-2",
+  "ISSN" : [ "0987-7983" ],
+  "issn-type" : [ { "value" : "0987-7983", "type" : "print" } ],
   "subject" : [ "Pediatrics, Perinatology, and Child Health" ]
 }
 """
+  // scalastyle:on
   val CrossrefStringWithTitle = CrossrefString.replace("<<TITLE>>", "SomeTitle")
   val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle")
   val MalformedCrossrefString = CrossrefString.replace("}", "")
@@ -168,7 +175,7 @@ class ScoreJobTest extends FlatSpec with Matchers {
       2 -> CrossrefStrings(2),
       3 -> CrossrefStrings(3)))
     .sink[(String, Int, String, String)](TypedTsv[(String, Int, String, String)](output)) {
-      // Grobid titles and slugs (in parentheses): 
+      // Grobid titles and slugs (in parentheses):
       //   Title 1                       (title1)
       //   Title 2: TNG                  (title2)
       //   Title 3: The Sequel           (title3)
@@ -178,7 +185,7 @@ class ScoreJobTest extends FlatSpec with Matchers {
       //   Title 1: TNG 3                (title1)
       //   Title 2: Rebooted             (title2)
       // Join should have 3 "title1" slugs and 1 "title2" slug
-      outputBuffer => 
+      outputBuffer =>
       "The pipeline" should "return a 4-element list" in {
         outputBuffer should have length 4
       }
@@ -190,9 +197,9 @@ class ScoreJobTest extends FlatSpec with Matchers {
         countMap("title2") shouldBe 1
       }
 
-      def bundle(slug : String, grobidIndex : Int, crossrefIndex : Int) = {
+      def bundle(slug : String, grobidIndex : Int, crossrefIndex : Int) : (String, Int, String, String) = {
         val mf1 : MapFeatures = GrobidScorable.jsonToMapFeatures(
-          Sha1Strings(grobidIndex), 
+          Sha1Strings(grobidIndex),
           JsonStrings(grobidIndex))
         val mf2 : MapFeatures = CrossrefScorable.jsonToMapFeatures(
           CrossrefStrings(crossrefIndex))
-- 
cgit v1.2.3