scorable: test for more punctuation removal

1 files changed, 8 insertions, 0 deletions

diff --git a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
index 5ffc305..fd01c91 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
@@ -34,10 +34,18 @@ class ScorableFeaturesTest extends FlatSpec with Matchers {
     titleToSlug("a:b:c") shouldBe "a"
     titleToSlug(
       "If you're happy and you know it, clap your hands!") shouldBe "ifyourehappyandyouknowitclapyourhands"
+    titleToSlug(":;\"\'") shouldBe Scorable.NoSlug
+  }
+
+  it should "strip special characters" in {
+    titleToSlug(":;!',|\"\'`.#?!-@*/\\=+~%$^{}()[]<>-_…") shouldBe Scorable.NoSlug
+    // TODO: titleToSlug("©™₨№") shouldBe Scorable.NoSlug
+    // TODO: titleToSlug("πµΣσ") shouldBe Scorable.NoSlug
   }
 
   it should "remove whitespace" in {
     titleToSlug("foo bar : baz ::") shouldBe "foobar"
     titleToSlug("\na\t:b:c") shouldBe "a"
+    titleToSlug("\n \t \r  ") shouldBe Scorable.NoSlug
   }
 }