From 96513b5cc26dd91375a6433e49d24b2d9eb2bea9 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 25 Jun 2020 22:02:04 -0700 Subject: pdfextract: handle too-large fulltext --- python/sandcrawler/pdfextract.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'python') diff --git a/python/sandcrawler/pdfextract.py b/python/sandcrawler/pdfextract.py index a6c25c1..d1b060e 100644 --- a/python/sandcrawler/pdfextract.py +++ b/python/sandcrawler/pdfextract.py @@ -198,6 +198,23 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr for n in range(1, pdf.pages): pageN = pdf.create_page(n) full_text += pageN.text() + + # Kafka message size limit; cap at about 1 MByte + if len(full_text)> 1000000: + return PdfExtractResult( + sha1hex=sha1hex, + status='text-too-large', + error_msg="full_text chars: {}".format(len(full_text)), + file_meta=file_meta, + ) + if len(pdf.metadata)> 1000000: + return PdfExtractResult( + sha1hex=sha1hex, + status='text-too-large', + error_msg="meta_xml chars: {}".format(len(full_text)), + file_meta=file_meta, + ) + pdf_info = pdf.infos() # TODO: is this actually needed? or does json marshalling work automatically? for k in pdf_info.keys(): -- cgit v1.2.3