diff options
author | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-07 10:11:54 -0700 |
---|---|---|
committer | Ellen Spertus <ellen.spertus@gmail.com> | 2018-08-07 10:11:54 -0700 |
commit | cbd6433af7949df7c4433468bf99eefe9973e864 (patch) | |
tree | 7ad3f0284581cd39652527ba07cd9bd4e7e860bf /scalding/src/test/scala | |
parent | 7eed53615e3a106d1cbf7cc451b74674fd2c3daa (diff) | |
download | sandcrawler-cbd6433af7949df7c4433468bf99eefe9973e864.tar.gz sandcrawler-cbd6433af7949df7c4433468bf99eefe9973e864.zip |
Removed commented-out code.
Diffstat (limited to 'scalding/src/test/scala')
-rw-r--r-- | scalding/src/test/scala/sandcrawler/ScorableTest.scala | 108 |
1 files changed, 108 insertions, 0 deletions
diff --git a/scalding/src/test/scala/sandcrawler/ScorableTest.scala b/scalding/src/test/scala/sandcrawler/ScorableTest.scala new file mode 100644 index 0000000..0375b6a --- /dev/null +++ b/scalding/src/test/scala/sandcrawler/ScorableTest.scala @@ -0,0 +1,108 @@ +package sandcrawler + +import cascading.tuple.Fields +import cascading.tuple.Tuple +import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions} +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import org.apache.hadoop.hbase.util.Bytes +import org.scalatest._ +import parallelai.spyglass.hbase.HBaseConstants.SourceMode + +class HBaseCrossrefScoreTest extends FlatSpec with Matchers { + val JsonString = """ +{ + "title": "<<TITLE>>", + "authors": [ + {"name": "Brewster Kahle"}, + {"name": "J Doe"} + ], + "journal": { + "name": "Dummy Example File. Journal of Fake News. pp. 1-2. ISSN 1234-5678", + "eissn": null, + "issn": null, + "issue": null, + "publisher": null, + "volume": null + }, + "date": "2000", + "doi": null, + "citations": [ + { "authors": [{"name": "A Seaperson"}], + "date": "2001", + "id": "b0", + "index": 0, + "issue": null, + "journal": "Letters in the Alphabet", + "publisher": null, + "title": "Everything is Wonderful", + "url": null, + "volume": "20"}, + { "authors": [], + "date": "2011-03-28", + "id": "b1", + "index": 1, + "issue": null, + "journal": "The Dictionary", + "publisher": null, + "title": "All about Facts", + "url": null, + "volume": "14"} + ], + "abstract": "Everything you ever wanted to know about nothing", + "body": "Introduction \nEverything starts somewhere, as somebody [1] once said. \n\n In Depth \n Meat \nYou know, for kids. \n Potatos \nQED.", + "acknowledgement": null, + "annex": null +} +""" + val MalformedJsonString = JsonString.replace("}", "") + + "titleToSlug()" should "extract the parts of titles before a colon" in { + val slug = Scorable.titleToSlug("HELLO:there") + slug should contain ("hello") + } + + it should "extract an entire colon-less string" in { + val slug = Scorable.titleToSlug("hello THERE") + slug should contain ("hello there") + } + + it should "return None if given empty string" in { + Scorable.titleToSlug("") shouldBe None + } + + "jsonToMap()" should "return a map, given a legal JSON string" in { + Scorable.jsonToMap(jsonString) should be (Some(_)) + } + + it should "return None, given illegal JSON" in { + Scorable.jsonToMap("illegal{,json{{") should be (None)) + } + +/* + it should "return None if given a malformed json string" in { + val slug = Scorable.grobidToSlug(MalformedGrobidString) + slug shouldBe None + } + + it should "return None if given an empty json string" in { + val slug = Scorable.grobidToSlug("") + slug shouldBe None + } + + "crossrefToSlug()" should "get the right slug for a crossref json string" in { + val slug = Scorable.crossrefToSlug(CrossrefStringWithTitle) + slug should contain ("sometitle") + } + + it should "return None if given json string without title" in { + val slug = Scorable.grobidToSlug(CrossrefStringWithoutTitle) + slug shouldBe None + } + + it should "return None if given a malformed json string" in { + val slug = Scorable.grobidToSlug(MalformedCrossrefString) + slug shouldBe None + } + */ +} + |