diff options
author | bnewbold <bnewbold@archive.org> | 2018-08-16 21:09:54 +0000 |
---|---|---|
committer | bnewbold <bnewbold@archive.org> | 2018-08-16 21:09:54 +0000 |
commit | af0fa6edf3c21ac38a8ab4e0fb425e5471e6c3b6 (patch) | |
tree | d32953c30c5f7342672694708b385936e5a36dfe /scalding/src/test/scala/sandcrawler/ScorableTest.scala | |
parent | 71be2e685848a31888811e2e398e769f7e0486c2 (diff) | |
parent | 96ea0ddd06ee4a7c11c7d5def976749ab3675878 (diff) | |
download | sandcrawler-af0fa6edf3c21ac38a8ab4e0fb425e5471e6c3b6.tar.gz sandcrawler-af0fa6edf3c21ac38a8ab4e0fb425e5471e6c3b6.zip |
Merge branch 'bnewbold-scoring-patches' into 'master'
Patches on top of scoring-refactor branch (Crossref/GROBID matching work)
See merge request webgroup/sandcrawler!15
Diffstat (limited to 'scalding/src/test/scala/sandcrawler/ScorableTest.scala')
-rw-r--r-- | scalding/src/test/scala/sandcrawler/ScorableTest.scala | 73 |
1 files changed, 73 insertions, 0 deletions
diff --git a/scalding/src/test/scala/sandcrawler/ScorableTest.scala b/scalding/src/test/scala/sandcrawler/ScorableTest.scala new file mode 100644 index 0000000..f63bef8 --- /dev/null +++ b/scalding/src/test/scala/sandcrawler/ScorableTest.scala @@ -0,0 +1,73 @@ +package sandcrawler + +import cascading.tuple.Fields +import cascading.tuple.Tuple +import com.twitter.scalding.JobTest +import com.twitter.scalding.TextLine +import com.twitter.scalding.TupleConversions +import com.twitter.scalding.TypedTsv +import org.apache.hadoop.hbase.io.ImmutableBytesWritable +import org.apache.hadoop.hbase.util.Bytes +import org.scalatest._ +import parallelai.spyglass.hbase.HBaseConstants.SourceMode + +class ScorableTest extends FlatSpec with Matchers { + val JsonString = """ +{ + "title": "<<TITLE>>", + "authors": [ + {"name": "Brewster Kahle"}, + {"name": "J Doe"} + ], + "journal": { + "name": "Dummy Example File. Journal of Fake News. pp. 1-2. ISSN 1234-5678", + "eissn": null, + "issn": null, + "issue": null, + "publisher": null, + "volume": null + }, + "date": "2000", + "doi": null, + "citations": [ + { "authors": [{"name": "A Seaperson"}], + "date": "2001", + "id": "b0", + "index": 0, + "issue": null, + "journal": "Letters in the Alphabet", + "publisher": null, + "title": "Everything is Wonderful", + "url": null, + "volume": "20"}, + { "authors": [], + "date": "2011-03-28", + "id": "b1", + "index": 1, + "issue": null, + "journal": "The Dictionary", + "publisher": null, + "title": "All about Facts", + "url": null, + "volume": "14"} + ], + "abstract": "Everything you ever wanted to know about nothing", + "body": "Introduction \nEverything starts somewhere, as somebody [1] once said. \n\n In Depth \n Meat \nYou know, for kids. \n Potatos \nQED.", + "acknowledgement": null, + "annex": null +} +""" + "jsonToMap()" should "return a map, given a legal JSON string" in { + Scorable.jsonToMap(JsonString) should not be (None) + } + + it should "return None, given illegal JSON" in { + Scorable.jsonToMap("illegal{,json{{") should be (None) + } + + "computeOutput()" should "return Scorable.MaxScore if given identical ReduceFeatures" in { + val score = Scorable.computeSimilarity( + new ReduceFeatures(JsonString), new ReduceFeatures(JsonString)) + score shouldBe Scorable.MaxScore + } +} |