diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2019-12-27 18:11:10 -0800 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-01-02 18:12:58 -0800 |
| commit | 6093c9a0c9b65cdf790f200395e2d44d4fe6278b (patch) | |
| tree | 967145462d5804e05195088fb56eb6374e3a2c34 /python/sandcrawler/persist.py | |
| parent | 2a7a51e9ee154170ad6d48b36281f6ee702da582 (diff) | |
| download | sandcrawler-6093c9a0c9b65cdf790f200395e2d44d4fe6278b.tar.gz sandcrawler-6093c9a0c9b65cdf790f200395e2d44d4fe6278b.zip | |
remove unused filter in grobid worker
Diffstat (limited to 'python/sandcrawler/persist.py')
| -rw-r--r-- | python/sandcrawler/persist.py | 1 |
1 files changed, 0 insertions, 1 deletions
diff --git a/python/sandcrawler/persist.py b/python/sandcrawler/persist.py index 45df8c7..c3f6b08 100644 --- a/python/sandcrawler/persist.py +++ b/python/sandcrawler/persist.py @@ -256,7 +256,6 @@ class PersistGrobidWorker(SandcrawlerWorker): r['metadata'] = metadata if not self.s3_only: - grobid_batch = [r['grobid'] for r in batch if r.get('grobid')] resp = self.db.insert_grobid(self.cur, batch, on_conflict="update") self.counts['insert-grobid'] += resp[0] self.counts['update-grobid'] += resp[1] |