From 98e67e291132b10a0ca698ad4ff754acc0c22121 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 23 Aug 2018 19:35:23 -0700 Subject: add a content-type filter for crossref works --- scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'scalding/src/test') diff --git a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala index 0cb12ee..f598cae 100644 --- a/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala +++ b/scalding/src/test/scala/sandcrawler/CrossrefScorableTest.scala @@ -74,6 +74,8 @@ class CrossrefScorableTest extends FlatSpec with Matchers { val CrossrefStringWithoutTitle = CrossrefString.replace("title", "nottitle") val MalformedCrossrefString = CrossrefString.replace("}", "") val CrossrefStringWithNoAuthors = CrossrefString.replace("<>", "Some Valid Title").replace("author", "no-author") + val CrossrefStringWrongType = CrossrefString.replace("<<TITLE>>", "Some Valid Title").replace("journal-article", "other") + val CrossrefStringNoType = CrossrefString.replace("<<TITLE>>", "Some Valid Title").replace("type", "not-type") // Unit tests "CrossrefScorable.jsonToMapFeatures()" should "handle invalid JSON" in { @@ -139,4 +141,11 @@ class CrossrefScorableTest extends FlatSpec with Matchers { it should "return false for invalid JSON" in { CrossrefScorable.keepRecord(CrossrefStringWithoutTitle) shouldBe false } + + it should "handle content types" in { + val resultWrong = CrossrefScorable.jsonToMapFeatures(CrossrefStringWrongType) + resultWrong.slug shouldBe Scorable.NoSlug + val resultMissing = CrossrefScorable.jsonToMapFeatures(CrossrefStringNoType) + resultMissing.slug shouldBe Scorable.NoSlug + } } -- cgit v1.2.3