aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src/test
diff options
context:
space:
mode:
authorEllen Spertus <ellen.spertus@gmail.com>2018-08-07 10:11:54 -0700
committerEllen Spertus <ellen.spertus@gmail.com>2018-08-07 10:11:54 -0700
commitcbd6433af7949df7c4433468bf99eefe9973e864 (patch)
tree7ad3f0284581cd39652527ba07cd9bd4e7e860bf /scalding/src/test
parent7eed53615e3a106d1cbf7cc451b74674fd2c3daa (diff)
downloadsandcrawler-cbd6433af7949df7c4433468bf99eefe9973e864.tar.gz
sandcrawler-cbd6433af7949df7c4433468bf99eefe9973e864.zip
Removed commented-out code.
Diffstat (limited to 'scalding/src/test')
-rw-r--r--scalding/src/test/scala/sandcrawler/ScorableTest.scala108
1 files changed, 108 insertions, 0 deletions
diff --git a/scalding/src/test/scala/sandcrawler/ScorableTest.scala b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
new file mode 100644
index 0000000..0375b6a
--- /dev/null
+++ b/scalding/src/test/scala/sandcrawler/ScorableTest.scala
@@ -0,0 +1,108 @@
+package sandcrawler
+
+import cascading.tuple.Fields
+import cascading.tuple.Tuple
+import com.twitter.scalding.{JobTest, TextLine, TypedTsv, TupleConversions}
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable
+import org.apache.hadoop.hbase.util.Bytes
+import org.scalatest._
+import parallelai.spyglass.hbase.HBaseConstants.SourceMode
+
+class HBaseCrossrefScoreTest extends FlatSpec with Matchers {
+ val JsonString = """
+{
+ "title": "<<TITLE>>",
+ "authors": [
+ {"name": "Brewster Kahle"},
+ {"name": "J Doe"}
+ ],
+ "journal": {
+ "name": "Dummy Example File. Journal of Fake News. pp. 1-2. ISSN 1234-5678",
+ "eissn": null,
+ "issn": null,
+ "issue": null,
+ "publisher": null,
+ "volume": null
+ },
+ "date": "2000",
+ "doi": null,
+ "citations": [
+ { "authors": [{"name": "A Seaperson"}],
+ "date": "2001",
+ "id": "b0",
+ "index": 0,
+ "issue": null,
+ "journal": "Letters in the Alphabet",
+ "publisher": null,
+ "title": "Everything is Wonderful",
+ "url": null,
+ "volume": "20"},
+ { "authors": [],
+ "date": "2011-03-28",
+ "id": "b1",
+ "index": 1,
+ "issue": null,
+ "journal": "The Dictionary",
+ "publisher": null,
+ "title": "All about Facts",
+ "url": null,
+ "volume": "14"}
+ ],
+ "abstract": "Everything you ever wanted to know about nothing",
+ "body": "Introduction \nEverything starts somewhere, as somebody [1] once said. \n\n In Depth \n Meat \nYou know, for kids. \n Potatos \nQED.",
+ "acknowledgement": null,
+ "annex": null
+}
+"""
+ val MalformedJsonString = JsonString.replace("}", "")
+
+ "titleToSlug()" should "extract the parts of titles before a colon" in {
+ val slug = Scorable.titleToSlug("HELLO:there")
+ slug should contain ("hello")
+ }
+
+ it should "extract an entire colon-less string" in {
+ val slug = Scorable.titleToSlug("hello THERE")
+ slug should contain ("hello there")
+ }
+
+ it should "return None if given empty string" in {
+ Scorable.titleToSlug("") shouldBe None
+ }
+
+ "jsonToMap()" should "return a map, given a legal JSON string" in {
+ Scorable.jsonToMap(jsonString) should be (Some(_))
+ }
+
+ it should "return None, given illegal JSON" in {
+ Scorable.jsonToMap("illegal{,json{{") should be (None))
+ }
+
+/*
+ it should "return None if given a malformed json string" in {
+ val slug = Scorable.grobidToSlug(MalformedGrobidString)
+ slug shouldBe None
+ }
+
+ it should "return None if given an empty json string" in {
+ val slug = Scorable.grobidToSlug("")
+ slug shouldBe None
+ }
+
+ "crossrefToSlug()" should "get the right slug for a crossref json string" in {
+ val slug = Scorable.crossrefToSlug(CrossrefStringWithTitle)
+ slug should contain ("sometitle")
+ }
+
+ it should "return None if given json string without title" in {
+ val slug = Scorable.grobidToSlug(CrossrefStringWithoutTitle)
+ slug shouldBe None
+ }
+
+ it should "return None if given a malformed json string" in {
+ val slug = Scorable.grobidToSlug(MalformedCrossrefString)
+ slug shouldBe None
+ }
+ */
+}
+