aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--python/sandcrawler/pdfextract.py34
1 files changed, 29 insertions, 5 deletions
diff --git a/python/sandcrawler/pdfextract.py b/python/sandcrawler/pdfextract.py
index c950d18..4e55f3f 100644
--- a/python/sandcrawler/pdfextract.py
+++ b/python/sandcrawler/pdfextract.py
@@ -65,8 +65,24 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr
try:
pdf = poppler.load_from_data(blob)
+ if pdf is None:
+ return PdfExtractResult(
+ sha1hex=sha1hex,
+ status='empty-pdf',
+ file_meta=file_meta,
+ )
page0 = pdf.create_page(0)
- except NotImplementedError as e:
+ if page0 is None:
+ return PdfExtractResult(
+ sha1hex=sha1hex,
+ status='empty-page0',
+ file_meta=file_meta,
+ )
+ # this call sometimes fails an returns an AttributeError
+ page0rect = page0.page_rect()
+ except AttributeError as e:
+ # may need to expand the set of exceptions caught here over time, but
+ # starting with a narrow set
return PdfExtractResult(
sha1hex=sha1hex,
status='parse-error',
@@ -74,6 +90,7 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr
file_meta=file_meta,
)
+ assert page0 is not None
page0_thumbnail: Optional[bytes] = None
renderer = poppler.PageRenderer()
try:
@@ -90,7 +107,6 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr
print(str(e), file=sys.stderr)
page0_thumbnail = None
- page0rect = page0.page_rect()
full_text = page0.text()
for n in range(1, pdf.pages):
pageN = pdf.create_page(n)
@@ -101,6 +117,14 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr
if isinstance(pdf_info[k], datetime.datetime):
pdf_info[k] = datetime.datetime.isoformat(pdf_info[k])
+ permanent_id: Optional[str] = None
+ update_id: Optional[str] = None
+ try:
+ permanent_id = pdf.pdf_id.permanent_id
+ update_id = pdf.pdf_id.update_id
+ except TypeError:
+ pass
+
return PdfExtractResult(
sha1hex=sha1hex,
file_meta=file_meta,
@@ -109,13 +133,13 @@ def process_pdf(blob: bytes, thumb_size=(180,300), thumb_type="JPEG") -> PdfExtr
text=full_text or None,
page0_thumbnail=page0_thumbnail,
meta_xml=pdf.metadata or None,
- pdf_info=pdf.infos(),
+ pdf_info=pdf_info,
pdf_extra=dict(
height=page0rect.height,
width=page0rect.width,
page_count=pdf.pages,
- permanent_id=pdf.pdf_id.permanent_id,
- update_id=pdf.pdf_id.update_id,
+ permanent_id=permanent_id,
+ update_id=update_id,
pdf_version=f"{pdf.pdf_version[0]}.{pdf.pdf_version[1]}",
),
)