blacklist -> denylist

5 files changed, 12 insertions, 12 deletions

diff --git a/python/extraction_cdx_grobid.py b/python/extraction_cdx_grobid.py
index 040538c..b27d053 100755
--- a/python/extraction_cdx_grobid.py
+++ b/python/extraction_cdx_grobid.py
@@ -37,7 +37,7 @@ from grobid2json import teixml2json
 sentry_client = raven.Client()
 
 # Specific poison-pill rows we should skip
-KEY_BLACKLIST = (
+KEY_DENYLIST = (
     'sha1:DLCCSMMVTCCIR6LRXHEQLZ4PWO6NG2YT',    # "failed to guess ARC header format"
 )
 
@@ -210,9 +210,9 @@ class MRExtractCdxGrobid(MRJob):
             yield _, status
             return
         key = info['key']
-        if key in KEY_BLACKLIST:
-            self.increment_counter('lines', 'blacklist')
-            yield _, dict(status='blacklist', key=key)
+        if key in KEY_DENYLIST:
+            self.increment_counter('lines', 'denylist')
+            yield _, dict(status='denylist', key=key)
             return
 
         # Note: this may not get "cleared" correctly
diff --git a/python/extraction_ungrobided.py b/python/extraction_ungrobided.py
index 4074112..972b8f9 100755
--- a/python/extraction_ungrobided.py
+++ b/python/extraction_ungrobided.py
@@ -37,7 +37,7 @@ from grobid2json import teixml2json
 sentry_client = raven.Client()
 
 # Specific poison-pill rows we should skip
-KEY_BLACKLIST = (
+KEY_DENYLIST = (
     'sha1:DLCCSMMVTCCIR6LRXHEQLZ4PWO6NG2YT',    # "failed to guess ARC header format"
 )
 
@@ -212,9 +212,9 @@ class MRExtractUnGrobided(MRJob):
             yield _, status
             return
         key = info['key']
-        if key in KEY_BLACKLIST:
-            self.increment_counter('lines', 'blacklist')
-            yield _, dict(status='blacklist', key=key)
+        if key in KEY_DENYLIST:
+            self.increment_counter('lines', 'denylist')
+            yield _, dict(status='denylist', key=key)
             return
 
         # Note: this may not get "cleared" correctly
diff --git a/scalding/src/main/resources/slug-blacklist.txt b/scalding/src/main/resources/slug-denylist.txt
index 926dbd5..926dbd5 100644
--- a/scalding/src/main/resources/slug-blacklist.txt
+++ b/scalding/src/main/resources/slug-denylist.txt
diff --git a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
index be2b495..95a39aa 100644
--- a/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
+++ b/scalding/src/main/scala/sandcrawler/ScorableFeatures.scala
@@ -8,8 +8,8 @@ import scala.util.parsing.json.JSONObject
 
 object ScorableFeatures {
   // TODO: Add exception handling.
-  val fileStream : InputStream = getClass.getResourceAsStream("/slug-blacklist.txt")
-  val SlugBlacklist : Set[String] = Source.fromInputStream(fileStream).getLines.toSet
+  val fileStream : InputStream = getClass.getResourceAsStream("/slug-denylist.txt")
+  val SlugDenylist : Set[String] = Source.fromInputStream(fileStream).getLines.toSet
   fileStream.close
   val MinSlugLength = 8
 
@@ -44,7 +44,7 @@ class ScorableFeatures private(title : String, authors : List[Any] = List(), yea
       val slug = StringUtilities.removePunctuation((unaccented.toLowerCase())).replaceAll("\\s", "")
       if (slug.isEmpty
         || slug == null
-        || (ScorableFeatures.SlugBlacklist contains slug)
+        || (ScorableFeatures.SlugDenylist contains slug)
         || (slug.length < ScorableFeatures.MinSlugLength)) {
         None
       } else {
diff --git a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
index 112a5e5..c847296 100644
--- a/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
+++ b/scalding/src/test/scala/sandcrawler/ScorableFeaturesTest.scala
@@ -42,7 +42,7 @@ class ScorableFeaturesTest extends FlatSpec with Matchers {
   it should "filter stub titles" in {
     titleToSlug("abstract") shouldBe (None)
     titleToSlug("title!") shouldBe (None)
-    titleToSlug("a real title which is not on blacklist") shouldBe Some("arealtitlewhichisnotonblacklist")
+    titleToSlug("a real title which is not on denylist") shouldBe Some("arealtitlewhichisnotondenylist")
   }
 
   it should "strip special characters" in {