aboutsummaryrefslogtreecommitdiffstats
path: root/python/grobid_tool.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-01-14 16:05:41 -0800
committerBryan Newbold <bnewbold@archive.org>2020-01-14 16:05:41 -0800
commitba6f16a02cfde0e4acb499c00b456b42472c0b00 (patch)
tree9b27d9982e61ee97a82181249c0f9723980a4beb /python/grobid_tool.py
parent21599839802b8ef3a84ffe90855f7bceaaa12a0d (diff)
downloadsandcrawler-ba6f16a02cfde0e4acb499c00b456b42472c0b00.tar.gz
sandcrawler-ba6f16a02cfde0e4acb499c00b456b42472c0b00.zip
more ftp status 226 support
Diffstat (limited to 'python/grobid_tool.py')
-rwxr-xr-xpython/grobid_tool.py18
1 files changed, 13 insertions, 5 deletions
diff --git a/python/grobid_tool.py b/python/grobid_tool.py
index ad7841d..dc9780d 100755
--- a/python/grobid_tool.py
+++ b/python/grobid_tool.py
@@ -36,13 +36,21 @@ def run_extract_cdx(args):
if args.jobs > 1:
worker = GrobidWorker(grobid_client, wayback_client, sink=None)
multi_worker = MultiprocessWrapper(worker, args.sink)
- pusher = CdxLinePusher(multi_worker, args.cdx_file,
- filter_http_statuses=[200], filter_mimetypes=['application/pdf'],
- batch_size=args.jobs)
+ pusher = CdxLinePusher(
+ multi_worker,
+ args.cdx_file,
+ filter_http_statuses=[200, 226],
+ filter_mimetypes=['application/pdf'],
+ batch_size=args.jobs,
+ )
else:
worker = GrobidWorker(grobid_client, wayback_client, sink=args.sink)
- pusher = CdxLinePusher(worker, args.cdx_file,
- filter_http_statuses=[200], filter_mimetypes=['application/pdf'])
+ pusher = CdxLinePusher(
+ worker,
+ args.cdx_file,
+ filter_http_statuses=[200, 226],
+ filter_mimetypes=['application/pdf'],
+ )
pusher.run()
def run_extract_zipfile(args):