From 3287f08a788107815f366019060a7cbcfe9505d2 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 17 Jun 2020 11:10:36 -0700 Subject: workers: refactor to pass key to process() --- python/sandcrawler/grobid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'python/sandcrawler/grobid.py') diff --git a/python/sandcrawler/grobid.py b/python/sandcrawler/grobid.py index d9db6c3..11623c5 100644 --- a/python/sandcrawler/grobid.py +++ b/python/sandcrawler/grobid.py @@ -95,7 +95,7 @@ class GrobidWorker(SandcrawlerFetchWorker): key=default_key, ) - def process(self, record): + def process(self, record, key=None): default_key = record['sha1hex'] fetch_result = self.fetch_blob(record) @@ -121,7 +121,7 @@ class GrobidBlobWorker(SandcrawlerWorker): self.sink = sink self.consolidate_mode = 2 - def process(self, blob): + def process(self, blob, key=None): if not blob: return None result = self.grobid_client.process_fulltext(blob, consolidate_mode=self.consolidate_mode) -- cgit v1.2.3