lots of grobid tool implementation (still WIP)

2 files changed, 29 insertions, 7 deletions

diff --git a/python/tests/test_grobid.py b/python/tests/test_grobid.py
index fca234a..10560cd 100644
--- a/python/tests/test_grobid.py
+++ b/python/tests/test_grobid.py
@@ -3,7 +3,7 @@ import pytest
 import struct
 import responses
 
-from sandcrawler import GrobidClient
+from sandcrawler import GrobidClient, GrobidWorker, CdxLinePusher, BlackholeSink, WaybackClient
 
 
 FAKE_PDF_BYTES = b"%PDF SOME JUNK" + struct.pack("!q", 112853843)
@@ -28,11 +28,10 @@ def test_grobid_503():
 
     assert resp['status_code'] == 503
     assert resp['status'] == "error"
-    print(resp)
-    assert False
 
 @responses.activate
-def test_grobid_503():
+@pytest.mark.skip(reason="XXX: need to fix unicode/bytes something something")
+def test_grobid_success():
 
     client = GrobidClient(host_url="http://localhost:8070")
 
@@ -51,3 +50,26 @@ def test_grobid_503():
     print(type(REAL_TEI_XML))
     assert resp['tei_xml'] == REAL_TEI_XML.decode('utf-8')
     #assert resp['tei_xml'].split('\n')[:3] == REAL_TEI_XML.split('\n')[:3]
+
+@responses.activate
+def test_grobid_worker_cdx():
+
+    sink = BlackholeSink()
+    grobid_client = GrobidClient(host_url="http://localhost:8070")
+    wayback_client = WaybackClient()
+    worker = GrobidWorker(grobid_client, wayback_client, sink=sink)
+
+    responses.add(responses.POST,
+        'http://localhost:8070/api/processFulltextDocument', status=200,
+        body=REAL_TEI_XML, content_type='text/xml')
+
+    with open('tests/files/example.cdx', 'r') as cdx_file:
+        pusher = CdxLinePusher(worker, cdx_file,
+            filter_http_statuses=[200], filter_mimetypes=['application/pdf'])
+        pusher_counts = pusher.run()
+        assert pusher_counts['total']
+        assert pusher_counts['pushed'] == 7
+        assert pusher_counts['pushed'] == worker.counts['total']
+
+    assert len(responses.calls) == worker.counts['total']
+
diff --git a/python/tests/test_misc.py b/python/tests/test_misc.py
index 02deec9..420bc07 100644
--- a/python/tests/test_misc.py
+++ b/python/tests/test_misc.py
@@ -50,9 +50,9 @@ def test_parse_cdx_line():
         'surt': "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf",
         'url': "https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf",
         'datetime': "20170828233154",
-        'warc': "SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz",
-        'offset': 931661233,
-        'c_size': 210251,
+        'warc_path': "SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz",
+        'warc_offset': 931661233,
+        'warc_csize': 210251,
         'http_status': 200,
     }