python tests for pusher classes

1 files changed, 28 insertions, 0 deletions

diff --git a/python/tests/test_pushers.py b/python/tests/test_pushers.py
new file mode 100644
index 0000000..ed9c0bb
--- /dev/null
+++ b/python/tests/test_pushers.py
@@ -0,0 +1,28 @@
+
+import pytest
+
+from sandcrawler.workers import CdxLinePusher, BlackholeSink
+
+
+def test_cdx_line_pusher():
+
+    sink = BlackholeSink()
+
+    # vanilla (only default filters)
+    with open('tests/files/example.cdx', 'r') as cdx_file:
+        pusher = CdxLinePusher(sink, cdx_file)
+        counts = pusher.run()
+    assert counts['total'] == 20
+    assert counts['skip-parse'] == 1
+    assert counts['pushed'] == 19
+
+    # HTTP 200 and application/pdf
+    with open('tests/files/example.cdx', 'r') as cdx_file:
+        pusher = CdxLinePusher(sink, cdx_file,
+            filter_mimetypes=['application/pdf'], filter_http_statuses=[200])
+        counts = pusher.run()
+    assert counts['total'] == 20
+    assert counts['skip-parse'] == 1
+    assert counts['skip-http_status'] == 10
+    assert counts['skip-mimetype'] == 2
+    assert counts['pushed'] == 7