aboutsummaryrefslogtreecommitdiffstats
path: root/mapreduce/tests/test_backfill_hbase_from_cdx.py
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@archive.org>2018-04-05 17:18:52 -0700
committerBryan Newbold <bnewbold@archive.org>2018-04-05 17:18:52 -0700
commit77577da13afe07b5177452122f4cee77e3357b4e (patch)
treed37e411121abb805c272f48a6f4579e7af650bb1 /mapreduce/tests/test_backfill_hbase_from_cdx.py
parent37a775851b2d21f2afe7418a8628c50ae37edc5b (diff)
downloadsandcrawler-77577da13afe07b5177452122f4cee77e3357b4e.tar.gz
sandcrawler-77577da13afe07b5177452122f4cee77e3357b4e.zip
improve test coverage
Diffstat (limited to 'mapreduce/tests/test_backfill_hbase_from_cdx.py')
-rw-r--r--mapreduce/tests/test_backfill_hbase_from_cdx.py19
1 files changed, 19 insertions, 0 deletions
diff --git a/mapreduce/tests/test_backfill_hbase_from_cdx.py b/mapreduce/tests/test_backfill_hbase_from_cdx.py
index 1a13e5b..2dbbc25 100644
--- a/mapreduce/tests/test_backfill_hbase_from_cdx.py
+++ b/mapreduce/tests/test_backfill_hbase_from_cdx.py
@@ -52,3 +52,22 @@ com,pbworks,educ333b)/robots.txt 20170705063311 http://educ333b.pbworks.com/robo
f_c = json.loads(row[b'f:c'].decode('utf-8'))
assert f_c['u'] == "http://cadmus.eui.eu/bitstream/handle/1814/36635/RSCAS_2015_03.pdf%3Bjsessionid%3D761393014319A39F40D32AE3EB3A853F?sequence%3D1"
assert b'i' not in f_c
+
+def test_parse_cdx_skip(job):
+
+ job.mapper_init()
+
+ print("CDX prefix")
+ raw = " com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf application/pdf 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz"
+ info, status = job.mapper(None, raw).__next__()
+ assert info is None
+ assert status['status'] == "invalid"
+ assert 'prefix' in status['reason']
+
+ print("mimetype")
+ raw = "com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf text/html 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705061647307-00039-00048-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz"
+ info, status = job.mapper(None, raw).__next__()
+ assert info is None
+ assert status['status'] == "skip"
+ assert 'mimetype' in status['reason']
+