Merge branch 'ellen-length-filtering' into 'master'

Filtering titles by length

See merge request webgroup/sandcrawler!21

3 files changed, 82 insertions, 11 deletions

diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
index 1789d1a..3d18a21 100644
--- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala
@@ -66,7 +66,10 @@ class CrossrefScorableTest extends FlatSpec with Matchers {
 }
 """
   // scalastyle:on
-  val CrossrefStringWithTitle = CrossrefString.replace("<<TITLE>>", "Some Title")
+  val CrossrefStringWithGoodTitle = CrossrefString.replace("<<TITLE>>", "Some Title")
+  val CrossrefStringWithMaximumTitle = CrossrefString.replace("<<TITLE>>", "T" * Scorable.MaxTitleLength)
+  val CrossrefStringWithExcessiveTitle = CrossrefString.replace("<<TITLE>>", "T" * Scorable.MaxTitleLength + "0")
+  val CrossrefStringWithNullTitle = CrossrefString.replace("\"<<TITLE>>\"", "null")
   val CrossrefStringWithEmptyTitle = CrossrefString.replace("<<TITLE>>", "")
   val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle")
   val MalformedCrossrefString = CrossrefString.replace("}", "")
@@ -82,13 +85,18 @@ class CrossrefScorableTest extends FlatSpec with Matchers {
     result.slug shouldBe Scorable.NoSlug
   }
 
+  it should "handle null title" in {
+    val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithNullTitle)
+    result.slug shouldBe Scorable.NoSlug
+  }
+
   it should "handle empty title" in {
     val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithEmptyTitle)
     result.slug shouldBe Scorable.NoSlug
   }
 
   it should "handle valid input" in {
-    val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithTitle)
+    val result = CrossrefScorable.jsonToMapFeatures(CrossrefStringWithGoodTitle)
     result.slug shouldBe "sometitle"
     Scorable.jsonToMap(result.json) match {
       case None => fail()
@@ -97,4 +105,28 @@ class CrossrefScorableTest extends FlatSpec with Matchers {
       }
     }
   }
+
+  "CrossrefScorable.keepRecord()" should "return true for valid JSON with title" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithGoodTitle) shouldBe true
+  }
+
+  it should "return true for valid JSON with a title of maximum permitted length" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithMaximumTitle) shouldBe true
+  }
+
+  it should "return false for valid JSON with excessively long title" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithExcessiveTitle) shouldBe false
+  }
+
+  it should "return false for valid JSON with null title" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithNullTitle) shouldBe false
+  }
+
+  it should "return false for valid JSON with no title" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithoutTitle) shouldBe false
+  }
+
+  it should "return false for invalid JSON" in {
+    CrossrefScorable.keepRecord(CrossrefStringWithoutTitle) shouldBe false
+  }
 }
diff --git a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
index 661824b..6c45cc5 100644
--- a/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
+++ b/scalding/src/test/scala/sandcrawler/GrobidScorableTest.scala
@@ -57,7 +57,10 @@ class GrobidScorableTest extends FlatSpec with Matchers {
   "annex": null
 }
 """
-  val GrobidStringWithTitle = GrobidString.replace("<<TITLE>>", "Dummy Example File")
+  val GrobidStringWithGoodTitle = GrobidString.replace("<<TITLE>>", "Dummy Example File")
+  val GrobidStringWithMaximumTitle = GrobidString.replace("<<TITLE>>", "T" * Scorable.MaxTitleLength)
+  val GrobidStringWithExcessiveTitle = GrobidString.replace("<<TITLE>>", "T" * Scorable.MaxTitleLength + "0")
+  val GrobidStringWithNullTitle = GrobidString.replace("\"<<TITLE>>\"", "null")
   val GrobidStringWithoutTitle = GrobidString.replace("title", "nottitle")
   val MalformedGrobidString = GrobidString.replace("}", "")
   val Key = "Dummy Key"
@@ -69,13 +72,18 @@ class GrobidScorableTest extends FlatSpec with Matchers {
     result.slug shouldBe Scorable.NoSlug
   }
 
+  it should "handle null title" in {
+    val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithNullTitle)
+    result.slug shouldBe Scorable.NoSlug
+  }
+
   it should "handle missing title" in {
     val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithoutTitle)
     result.slug shouldBe Scorable.NoSlug
   }
 
   it should "handle valid input" in {
-    val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithTitle)
+    val result = GrobidScorable.jsonToMapFeatures(Key, GrobidStringWithGoodTitle)
     result.slug shouldBe "dummyexamplefile"
     Scorable.jsonToMap(result.json) match {
       case None => fail()
@@ -85,4 +93,28 @@ class GrobidScorableTest extends FlatSpec with Matchers {
       }
     }
   }
+
+  "GrobidScorable.keepRecord()" should "return true for valid JSON with title" in {
+    GrobidScorable.keepRecord(GrobidStringWithGoodTitle) shouldBe true
+  }
+
+  it should "return true for valid JSON with a title of maximum permitted length" in {
+    GrobidScorable.keepRecord(GrobidStringWithMaximumTitle) shouldBe true
+  }
+
+  it should "return false for valid JSON with excessively long title" in {
+    GrobidScorable.keepRecord(GrobidStringWithExcessiveTitle) shouldBe false
+  }
+
+  it should "return false for valid JSON with null title" in {
+    GrobidScorable.keepRecord(GrobidStringWithNullTitle) shouldBe false
+  }
+
+  it should "return false for valid JSON with no title" in {
+    GrobidScorable.keepRecord(GrobidStringWithoutTitle) shouldBe false
+  }
+
+  it should "return false for invalid JSON" in {
+    GrobidScorable.keepRecord(GrobidStringWithoutTitle) shouldBe false
+  }
 }
diff --git a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
index 35c31e5..85d141a 100644
--- a/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScoreJobTest.scala
@@ -117,6 +117,7 @@ class ScoreJobTest extends FlatSpec with Matchers {
 }
 """
   // scalastyle:on
+  val TooLongOfTitle = "X" * Scorable.MaxTitleLength + "Y"  // arbitrary long string
   val CrossrefStringWithTitle = CrossrefString.replace("<<TITLE>>", "SomeTitle")
   val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle")
   val MalformedCrossrefString = CrossrefString.replace("}", "")
@@ -124,7 +125,8 @@ class ScoreJobTest extends FlatSpec with Matchers {
     CrossrefString.replace("<<TITLE>>", "Title 2: TNG").replace("<<DOI>>", "DOI-0"),
     CrossrefString.replace("<<TITLE>>", "Title 1: TNG 2A").replace("<<DOI>>", "DOI-0.5"),
     CrossrefString.replace("<<TITLE>>", "Title 1: TNG 3").replace("<<DOI>>", "DOI-0.75"),
-    CrossrefString.replace("<<TITLE>>", "Title 2: Rebooted").replace("<<DOI>>", "DOI-1"))
+    CrossrefString.replace("<<TITLE>>", "Title 2: Rebooted").replace("<<DOI>>", "DOI-1"),
+    CrossrefString.replace("<<TITLE>>", TooLongOfTitle).replace("<<DOI>>", "DOI-1"))
 
   //  Pipeline tests
   val output = "/tmp/testOutput"
@@ -137,7 +139,8 @@ class ScoreJobTest extends FlatSpec with Matchers {
     "sha1:SDKUVHC3YNNEGH5WAG5ZAAXWAEBNX4WT",
     "sha1:35985C3YNNEGH5WAG5ZAAXWAEBNXJW56",
     "sha1:93187A85273589347598473894839443",
-    "sha1:024937534094897039547e9824382943")
+    "sha1:024937534094897039547e9824382943",
+    "sha1:93229759932857982837892347893892")
 
   val JsonStrings : List[String] = List(
     JsonString.replace("<<TITLE>>", "Title 1"),
@@ -147,13 +150,15 @@ class ScoreJobTest extends FlatSpec with Matchers {
     JsonString.replace("<<TITLE>>", "Title 1"),
     MalformedJsonString,
     // This will have bad status.
-    JsonString.replace("<<TITLE>>", "Title 2")
+    JsonString.replace("<<TITLE>>", "Title 2"),
+    // This is in both sources but too long.
+    JsonString.replace("<<TITLE>>", TooLongOfTitle)
   )
 
   // bnewbold: status codes aren't strings, they are uint64
   val Ok : Long = 200
   val Bad : Long = 400
-  val StatusCodes = List(Ok, Ok, Ok, Bad, Ok, Bad)
+  val StatusCodes = List(Ok, Ok, Ok, Bad, Ok, Bad, Ok)
 
   val SampleDataHead : List[Tuple] = (Sha1Strings, JsonStrings, StatusCodes)
     .zipped
@@ -181,19 +186,21 @@ class ScoreJobTest extends FlatSpec with Matchers {
       0 -> CrossrefStrings(0),
       1 -> CrossrefStrings(1),
       2 -> CrossrefStrings(2),
-      3 -> CrossrefStrings(3)))
-    .sink[(String, ReduceFeatures)](TypedTsv[(String, ReduceFeatures)](output + ".trapped")) {
-        _ => () }
+      3 -> CrossrefStrings(3),
+      4 -> CrossrefStrings(4)))
+    .sink[(String, ReduceFeatures)](TypedTsv[(String, ReduceFeatures)](output + ".trapped")) { _ => () }
     .sink[(String, Int, String, String)](TypedTsv[(String, Int, String, String)](output)) {
       // Grobid titles and slugs (in parentheses):
       //   Title 1                       (title1)
       //   Title 2: TNG                  (title2tng)
       //   Title 3: The Sequel           (title3thesequel)
+      //   <too long of a title>
       // crossref titles and slugs (in parentheses):
       //   Title 2: TNG                  (title2tng)
       //   Title 1: TNG 2A               (title1tng2a)
       //   Title 1: TNG 3                (title1tng3)
       //   Title 2: Rebooted             (title2rebooted)
+      //   <too long of a title>
       // XXX: Join should have 3 "title1" slugs and 1 "title2tng" slug
       outputBuffer =>
       "The pipeline" should "return a 1-element list" in {