aboutsummaryrefslogtreecommitdiffstats
path: root/python/tests/test_pdf.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2020-06-17 11:55:45 -0700
committerBryan Newbold <bnewbold@archive.org>2020-06-17 11:55:45 -0700
commitb58fa7ea1c0d077ac3e0fed18765726ab2b8da48 (patch)
treecfadb1a90df582dbd90cf1c90ec37d968d2039a1 /python/tests/test_pdf.py
parentdfec385e24629cd8ae9669cd91e091414bc7dcca (diff)
downloadsandcrawler-b58fa7ea1c0d077ac3e0fed18765726ab2b8da48.tar.gz
sandcrawler-b58fa7ea1c0d077ac3e0fed18765726ab2b8da48.zip
partial test coverage of pdf extract worker
Diffstat (limited to 'python/tests/test_pdf.py')
-rw-r--r--python/tests/test_pdf.py61
1 files changed, 61 insertions, 0 deletions
diff --git a/python/tests/test_pdf.py b/python/tests/test_pdf.py
new file mode 100644
index 0000000..1ccf85c
--- /dev/null
+++ b/python/tests/test_pdf.py
@@ -0,0 +1,61 @@
+
+import pytest
+import struct
+import responses
+
+from sandcrawler import PdfExtractWorker, PdfExtractBlobWorker, CdxLinePusher, BlackholeSink, WaybackClient
+from sandcrawler.pdf import process_pdf
+from test_wayback import wayback_client, cdx_client
+
+
+FAKE_PDF_BYTES = b"%PDF SOME JUNK" + struct.pack("!q", 112853843)
+
+def test_process_fake_pdf():
+ resp = process_pdf(FAKE_PDF_BYTES)
+ print(resp)
+ assert resp.status == "not-pdf"
+
+def test_process_dummy_pdf():
+ with open('tests/files/dummy.pdf', 'rb') as f:
+ pdf_bytes = f.read()
+ resp = process_pdf(pdf_bytes)
+ assert resp.status == 'success'
+ assert resp.page0_thumbnail is not None
+ assert len(resp.text) > 10
+ assert resp.meta_xml is None
+ assert resp.file_meta['mimetype'] == 'application/pdf'
+ print(resp.pdf_info)
+ print(resp.pdf_extra)
+ assert resp.pdf_info['Author'] == "Evangelos Vlachogiannis"
+ # 595 x 842
+ assert resp.pdf_extra['height'] == 842
+ assert resp.pdf_extra['width'] == 595
+ assert resp.pdf_extra['page_count'] == 1
+
+def test_pdfextract_worker_cdx(wayback_client):
+
+ sink = BlackholeSink()
+ worker = PdfExtractWorker(wayback_client, sink=sink, thumbnail_sink=sink)
+
+ with open('tests/files/example.cdx', 'r') as cdx_file:
+ pusher = CdxLinePusher(
+ worker,
+ cdx_file,
+ filter_http_statuses=[200, 226],
+ filter_mimetypes=['application/pdf'],
+ )
+ pusher_counts = pusher.run()
+ assert pusher_counts['total']
+ assert pusher_counts['pushed'] == 7
+ assert pusher_counts['pushed'] == worker.counts['total']
+
+def test_pdfextract_blob_worker():
+
+ sink = BlackholeSink()
+ worker = PdfExtractBlobWorker(sink=sink, thumbnail_sink=sink)
+
+ with open('tests/files/dummy.pdf', 'rb') as f:
+ pdf_bytes = f.read()
+
+ worker.process(pdf_bytes)
+