aboutsummaryrefslogtreecommitdiffstats
path: root/scalding/src
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-09-05 15:04:14 -0700
committerBryan Newbold <bnewbold@archive.org>2018-09-05 15:04:14 -0700
commit9d21a922386624b39b730849ab56c00a061f255f (patch)
tree836fe493d9fc5c531a1b2a020ae7d4b6d1384ea5 /scalding/src
parentcb8a3684867adceb9ac363799ef10be0ebca7850 (diff)
downloadsandcrawler-9d21a922386624b39b730849ab56c00a061f255f.tar.gz
sandcrawler-9d21a922386624b39b730849ab56c00a061f255f.zip
blacklist -> denylist
Diffstat (limited to 'scalding/src')
-rw-r--r--scalding/src/main/resources/slug-denylist.txt (renamed from scalding/src/main/resources/slug-blacklist.txt)0
-rw-r--r--scalding/src/main/scala/sandcrawler/ScorableFeatures.scala6
-rw-r--r--scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala2
3 files changed, 4 insertions, 4 deletions
diff --git a/scalding/src/main/resources/slug-blacklist.txt b/scalding/src/main/resources/slug-denylist.txt
index 926dbd5..926dbd5 100644
--- a/scalding/src/main/resources/slug-blacklist.txt
+++ b/scalding/src/main/resources/slug-denylist.txt
diff --git a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
index be2b495..95a39aa 100644
--- a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
+++ b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
@@ -8,8 +8,8 @@ import scala.util.parsing.json.JSONObject
object ScorableFeatures {
// TODO: Add exception handling.
- val fileStream : InputStream = getClass.getResourceAsStream("/slug-blacklist.txt")
- val SlugBlacklist : Set[String] = Source.fromInputStream(fileStream).getLines.toSet
+ val fileStream : InputStream = getClass.getResourceAsStream("/slug-denylist.txt")
+ val SlugDenylist : Set[String] = Source.fromInputStream(fileStream).getLines.toSet
fileStream.close
val MinSlugLength = 8
@@ -44,7 +44,7 @@ class ScorableFeatures private(title : String, authors : List[Any] = List(), yea
val slug = StringUtilities.removePunctuation((unaccented.toLowerCase())).replaceAll("\\s", "")
if (slug.isEmpty
|| slug == null
- || (ScorableFeatures.SlugBlacklist contains slug)
+ || (ScorableFeatures.SlugDenylist contains slug)
|| (slug.length < ScorableFeatures.MinSlugLength)) {
None
} else {
diff --git a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
index 112a5e5..c847296 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
@@ -42,7 +42,7 @@ class ScorableFeaturesTest extends FlatSpec with Matchers {
it should "filter stub titles" in {
titleToSlug("abstract") shouldBe (None)
titleToSlug("title!") shouldBe (None)
- titleToSlug("a real title which is not on blacklist") shouldBe Some("arealtitlewhichisnotonblacklist")
+ titleToSlug("a real title which is not on denylist") shouldBe Some("arealtitlewhichisnotondenylist")
}
it should "strip special characters" in {