diff options
Diffstat (limited to 'python/sandcrawler/workers.py')
| -rw-r--r-- | python/sandcrawler/workers.py | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/python/sandcrawler/workers.py b/python/sandcrawler/workers.py index 37e3d7a..d8a4016 100644 --- a/python/sandcrawler/workers.py +++ b/python/sandcrawler/workers.py @@ -1,16 +1,17 @@ -import sys import json -import time +import multiprocessing.pool import signal +import sys +import time import zipfile -import requests -import multiprocessing.pool from collections import Counter -from confluent_kafka import Consumer, Producer, KafkaException +import requests +from confluent_kafka import Consumer, KafkaException, Producer + +from .ia import PetaboxError, SandcrawlerBackoffError, WaybackContentError, WaybackError from .misc import parse_cdx_line -from .ia import SandcrawlerBackoffError, WaybackError, WaybackContentError, PetaboxError class SandcrawlerWorker(object): |