From 77577da13afe07b5177452122f4cee77e3357b4e Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 5 Apr 2018 17:18:52 -0700 Subject: improve test coverage --- mapreduce/tests/test_backfill_hbase_from_cdx.py | 19 +++++++++++++++++++ mapreduce/tests/test_common.py | 10 ++++++++++ mapreduce/tests/test_extraction_cdx_grobid.py | 1 - 3 files changed, 29 insertions(+), 1 deletion(-) (limited to 'mapreduce/tests') diff --git a/mapreduce/tests/test_backfill_hbase_from_cdx.py b/mapreduce/tests/test_backfill_hbase_from_cdx.py index 1a13e5b..2dbbc25 100644 --- a/mapreduce/tests/test_backfill_hbase_from_cdx.py +++ b/mapreduce/tests/test_backfill_hbase_from_cdx.py @@ -52,3 +52,22 @@ com,pbworks,educ333b)/robots.txt 20170705063311 http://educ333b.pbworks.com/robo f_c = json.loads(row[b'f:c'].decode('utf-8')) assert f_c['u'] == "http://cadmus.eui.eu/bitstream/handle/1814/36635/RSCAS_2015_03.pdf%3Bjsessionid%3D761393014319A39F40D32AE3EB3A853F?sequence%3D1" assert b'i' not in f_c + +def test_parse_cdx_skip(job): + + job.mapper_init() + + print("CDX prefix") + raw = " com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf application/pdf 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz" + info, status = job.mapper(None, raw).__next__() + assert info is None + assert status['status'] == "invalid" + assert 'prefix' in status['reason'] + + print("mimetype") + raw = "com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf text/html 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705061647307-00039-00048-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz" + info, status = job.mapper(None, raw).__next__() + assert info is None + assert status['status'] == "skip" + assert 'mimetype' in status['reason'] + diff --git a/mapreduce/tests/test_common.py b/mapreduce/tests/test_common.py index e2f96bb..34d50ed 100644 --- a/mapreduce/tests/test_common.py +++ b/mapreduce/tests/test_common.py @@ -28,3 +28,13 @@ def test_parse_cdx_line(): assert parse_cdx_line(raw) == correct assert parse_cdx_line(raw + "\n") == correct assert parse_cdx_line(raw + " extra_field") == correct + +def test_invalid_cdx(): + + print("missing warc") + raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 20170828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233 -" + assert parse_cdx_line(raw) == None + + print("bad datetime") + raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 2070828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233i SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz" + assert parse_cdx_line(raw) == None diff --git a/mapreduce/tests/test_extraction_cdx_grobid.py b/mapreduce/tests/test_extraction_cdx_grobid.py index 514a830..1d32c9f 100644 --- a/mapreduce/tests/test_extraction_cdx_grobid.py +++ b/mapreduce/tests/test_extraction_cdx_grobid.py @@ -121,7 +121,6 @@ def test_parse_cdx_skip(job): job.mapper_init() - print("warc format") raw = "com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf application/pdf 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz" info, status = job.mapper(None, raw).__next__() -- cgit v1.2.3