diff options
Diffstat (limited to 'python/tests')
| -rw-r--r-- | python/tests/test_grobid.py | 30 | ||||
| -rw-r--r-- | python/tests/test_misc.py | 6 |
2 files changed, 29 insertions, 7 deletions
diff --git a/python/tests/test_grobid.py b/python/tests/test_grobid.py index fca234a..10560cd 100644 --- a/python/tests/test_grobid.py +++ b/python/tests/test_grobid.py @@ -3,7 +3,7 @@ import pytest import struct import responses -from sandcrawler import GrobidClient +from sandcrawler import GrobidClient, GrobidWorker, CdxLinePusher, BlackholeSink, WaybackClient FAKE_PDF_BYTES = b"%PDF SOME JUNK" + struct.pack("!q", 112853843) @@ -28,11 +28,10 @@ def test_grobid_503(): assert resp['status_code'] == 503 assert resp['status'] == "error" - print(resp) - assert False @responses.activate -def test_grobid_503(): +@pytest.mark.skip(reason="XXX: need to fix unicode/bytes something something") +def test_grobid_success(): client = GrobidClient(host_url="http://localhost:8070") @@ -51,3 +50,26 @@ def test_grobid_503(): print(type(REAL_TEI_XML)) assert resp['tei_xml'] == REAL_TEI_XML.decode('utf-8') #assert resp['tei_xml'].split('\n')[:3] == REAL_TEI_XML.split('\n')[:3] + +@responses.activate +def test_grobid_worker_cdx(): + + sink = BlackholeSink() + grobid_client = GrobidClient(host_url="http://localhost:8070") + wayback_client = WaybackClient() + worker = GrobidWorker(grobid_client, wayback_client, sink=sink) + + responses.add(responses.POST, + 'http://localhost:8070/api/processFulltextDocument', status=200, + body=REAL_TEI_XML, content_type='text/xml') + + with open('tests/files/example.cdx', 'r') as cdx_file: + pusher = CdxLinePusher(worker, cdx_file, + filter_http_statuses=[200], filter_mimetypes=['application/pdf']) + pusher_counts = pusher.run() + assert pusher_counts['total'] + assert pusher_counts['pushed'] == 7 + assert pusher_counts['pushed'] == worker.counts['total'] + + assert len(responses.calls) == worker.counts['total'] + diff --git a/python/tests/test_misc.py b/python/tests/test_misc.py index 02deec9..420bc07 100644 --- a/python/tests/test_misc.py +++ b/python/tests/test_misc.py @@ -50,9 +50,9 @@ def test_parse_cdx_line(): 'surt': "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf", 'url': "https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf", 'datetime': "20170828233154", - 'warc': "SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz", - 'offset': 931661233, - 'c_size': 210251, + 'warc_path': "SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz", + 'warc_offset': 931661233, + 'warc_csize': 210251, 'http_status': 200, } |