From 6093c9a0c9b65cdf790f200395e2d44d4fe6278b Mon Sep 17 00:00:00 2001
From: Bryan Newbold <bnewbold@archive.org>
Date: Fri, 27 Dec 2019 18:11:10 -0800
Subject: remove unused filter in grobid worker

---
 python/sandcrawler/persist.py | 1 -
 1 file changed, 1 deletion(-)

(limited to 'python')

diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py
index 45df8c7..c3f6b08 100644
--- a/python/sandcrawler/persist.py
+++ b/python/sandcrawler/persist.py
@@ -256,7 +256,6 @@ class PersistGrobidWorker(SandcrawlerWorker):
             r['metadata'] = metadata
 
         if not self.s3_only:
-            grobid_batch = [r['grobid'] for r in batch if r.get('grobid')]
             resp = self.db.insert_grobid(self.cur, batch, on_conflict="update")
             self.counts['insert-grobid'] += resp[0]
             self.counts['update-grobid'] += resp[1]
-- 
cgit v1.2.3