diff options
| author | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 11:10:36 -0700 |
|---|---|---|
| committer | Bryan Newbold <bnewbold@archive.org> | 2020-06-17 11:10:36 -0700 |
| commit | 3287f08a788107815f366019060a7cbcfe9505d2 (patch) | |
| tree | 29a867d2cf84d116b26be37508d4ea6462dede88 /python/sandcrawler/ingest.py | |
| parent | 5a6bf449ac78586bf150216fe2310be178eeb6c3 (diff) | |
| download | sandcrawler-3287f08a788107815f366019060a7cbcfe9505d2.tar.gz sandcrawler-3287f08a788107815f366019060a7cbcfe9505d2.zip | |
workers: refactor to pass key to process()
Diffstat (limited to 'python/sandcrawler/ingest.py')
| -rw-r--r-- | python/sandcrawler/ingest.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/python/sandcrawler/ingest.py b/python/sandcrawler/ingest.py index 82b43fe..f4e78e4 100644 --- a/python/sandcrawler/ingest.py +++ b/python/sandcrawler/ingest.py @@ -23,7 +23,7 @@ class IngestFileWorker(SandcrawlerWorker): but is an HTML 200, treats it as a landing page, tries to extract fulltext link, then fetches that resource. - process(request) -> response + process(request, key=None) -> response Does all the things! Check existing processing (short circuit): @@ -243,7 +243,7 @@ class IngestFileWorker(SandcrawlerWorker): return False return True - def process(self, request): + def process(self, request, key=None): # backwards compatibility if request.get('ingest_type') in ('file', None): |