From 2d85d55ed1af4220f1469f4cf0cddd7e0888b3b3 Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Thu, 30 Apr 2020 16:46:44 -0700
Subject: workers: add missing want() dataflow path

---
 python/sandcrawler/workers.py | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'python')

diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py
index 96aef3f..6425e99 100644
--- a/python/sandcrawler/workers.py
+++ b/python/sandcrawler/workers.py
@@ -27,6 +27,9 @@ class SandcrawlerWorker(object):
 
     def push_record(self, task):
         self.counts['total'] += 1
+        if not self.want(task):
+            self.counts['skip'] += 1
+            return
         result = self.process(task)
         if not result:
             self.counts['failed'] += 1
@@ -90,6 +93,12 @@ class SandcrawlerWorker(object):
         print("Worker: {}".format(self.counts), file=sys.stderr)
         return self.counts
 
+    def want(self, task):
+        """
+        Optionally override this as a filter in implementations.
+        """
+        return True
+
     def process(self, task):
         """
         Derived workers need to implement business logic here.
-- 
cgit v1.2.3