From 910e59e9011935fecaae62520eb3fc30cbd65800 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 2 Oct 2019 18:02:40 -0700 Subject: python tests for pusher classes --- python/tests/files/dummy_zip.zip | Bin 0 -> 37760 bytes python/tests/test_pushers.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 python/tests/files/dummy_zip.zip create mode 100644 python/tests/test_pushers.py (limited to 'python') diff --git a/python/tests/files/dummy_zip.zip b/python/tests/files/dummy_zip.zip new file mode 100644 index 0000000..cb72dc8 Binary files /dev/null and b/python/tests/files/dummy_zip.zip differ diff --git a/python/tests/test_pushers.py b/python/tests/test_pushers.py new file mode 100644 index 0000000..ed9c0bb --- /dev/null +++ b/python/tests/test_pushers.py @@ -0,0 +1,28 @@ + +import pytest + +from sandcrawler.workers import CdxLinePusher, BlackholeSink + + +def test_cdx_line_pusher(): + + sink = BlackholeSink() + + # vanilla (only default filters) + with open('tests/files/example.cdx', 'r') as cdx_file: + pusher = CdxLinePusher(sink, cdx_file) + counts = pusher.run() + assert counts['total'] == 20 + assert counts['skip-parse'] == 1 + assert counts['pushed'] == 19 + + # HTTP 200 and application/pdf + with open('tests/files/example.cdx', 'r') as cdx_file: + pusher = CdxLinePusher(sink, cdx_file, + filter_mimetypes=['application/pdf'], filter_http_statuses=[200]) + counts = pusher.run() + assert counts['total'] == 20 + assert counts['skip-parse'] == 1 + assert counts['skip-http_status'] == 10 + assert counts['skip-mimetype'] == 2 + assert counts['pushed'] == 7 -- cgit v1.2.3