aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xmapreduce/backfill_hbase_from_cdx.py11
-rw-r--r--mapreduce/common.py2
-rwxr-xr-xmapreduce/extraction_cdx_grobid.py2
-rw-r--r--mapreduce/tests/test_backfill_hbase_from_cdx.py19
-rw-r--r--mapreduce/tests/test_common.py10
-rw-r--r--mapreduce/tests/test_extraction_cdx_grobid.py1
6 files changed, 39 insertions, 6 deletions
diff --git a/mapreduce/backfill_hbase_from_cdx.py b/mapreduce/backfill_hbase_from_cdx.py
index 8a28ec1..57c18e4 100755
--- a/mapreduce/backfill_hbase_from_cdx.py
+++ b/mapreduce/backfill_hbase_from_cdx.py
@@ -75,16 +75,19 @@ class MRCDXBackfillHBase(MRJob):
# Skip line
# XXX: tests don't cover this path; need coverage!
self.increment_counter('lines', 'invalid')
- return _, dict(status="invalid")
+ yield _, dict(status="invalid", reason="line prefix")
+ return
info = parse_cdx_line(raw_cdx)
if info is None:
self.increment_counter('lines', 'invalid')
- return _, dict(status="invalid")
+ yield _, dict(status="invalid")
+ return
if info['file:mime'] not in self.mime_filter:
self.increment_counter('lines', 'skip')
- return _, dict(status="skip")
+ yield _, dict(status="skip", reason="unwanted mimetype")
+ return
key = info.pop('key')
info['f:c'] = json.dumps(info['f:c'], sort_keys=True, indent=None)
@@ -96,6 +99,6 @@ class MRCDXBackfillHBase(MRJob):
yield _, dict(status="success")
-if __name__ == '__main__':
+if __name__ == '__main__': # pragma: no cover
MRCDXBackfillHBase.run()
diff --git a/mapreduce/common.py b/mapreduce/common.py
index 65b2744..1b8e572 100644
--- a/mapreduce/common.py
+++ b/mapreduce/common.py
@@ -29,6 +29,8 @@ def test_normalize_mime():
assert normalize_mime("Application/PDF") == "application/pdf"
assert normalize_mime("application/p") == None
assert normalize_mime("application/xml+stuff") == "text/xml"
+ assert normalize_mime("application/x-pdf") == "application/pdf"
+ assert normalize_mime("application/x-html") == None
def parse_cdx_line(raw_cdx):
diff --git a/mapreduce/extraction_cdx_grobid.py b/mapreduce/extraction_cdx_grobid.py
index c102a59..ea36e6e 100755
--- a/mapreduce/extraction_cdx_grobid.py
+++ b/mapreduce/extraction_cdx_grobid.py
@@ -207,6 +207,6 @@ class MRExtractCdxGrobid(MRJob):
yield _, dict(status="success", grobid_status=grobid_status)
-if __name__ == '__main__':
+if __name__ == '__main__': # pragma: no cover
MRExtractCdxGrobid.run()
diff --git a/mapreduce/tests/test_backfill_hbase_from_cdx.py b/mapreduce/tests/test_backfill_hbase_from_cdx.py
index 1a13e5b..2dbbc25 100644
--- a/mapreduce/tests/test_backfill_hbase_from_cdx.py
+++ b/mapreduce/tests/test_backfill_hbase_from_cdx.py
@@ -52,3 +52,22 @@ com,pbworks,educ333b)/robots.txt 20170705063311 http://educ333b.pbworks.com/robo
f_c = json.loads(row[b'f:c'].decode('utf-8'))
assert f_c['u'] == "http://cadmus.eui.eu/bitstream/handle/1814/36635/RSCAS_2015_03.pdf%3Bjsessionid%3D761393014319A39F40D32AE3EB3A853F?sequence%3D1"
assert b'i' not in f_c
+
+def test_parse_cdx_skip(job):
+
+ job.mapper_init()
+
+ print("CDX prefix")
+ raw = " com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf application/pdf 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz"
+ info, status = job.mapper(None, raw).__next__()
+ assert info is None
+ assert status['status'] == "invalid"
+ assert 'prefix' in status['reason']
+
+ print("mimetype")
+ raw = "com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf text/html 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705061647307-00039-00048-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz"
+ info, status = job.mapper(None, raw).__next__()
+ assert info is None
+ assert status['status'] == "skip"
+ assert 'mimetype' in status['reason']
+
diff --git a/mapreduce/tests/test_common.py b/mapreduce/tests/test_common.py
index e2f96bb..34d50ed 100644
--- a/mapreduce/tests/test_common.py
+++ b/mapreduce/tests/test_common.py
@@ -28,3 +28,13 @@ def test_parse_cdx_line():
assert parse_cdx_line(raw) == correct
assert parse_cdx_line(raw + "\n") == correct
assert parse_cdx_line(raw + " extra_field") == correct
+
+def test_invalid_cdx():
+
+ print("missing warc")
+ raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 20170828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233 -"
+ assert parse_cdx_line(raw) == None
+
+ print("bad datetime")
+ raw = "edu,upenn,ldc)/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf 2070828233154 https://www.ldc.upenn.edu/sites/www.ldc.upenn.edu/files/medar2009-large-arabic-broadcast-collection.pdf application/pdf 200 WL3FEA62TEU4F52Y5DOVQ62VET4QJW7G - - 210251 931661233i SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828231135742-00000-00009-wbgrp-svc284/SEMSCHOLAR-PDF-CRAWL-2017-08-04-20170828232253025-00005-3480~wbgrp-svc284.us.archive.org~8443.warc.gz"
+ assert parse_cdx_line(raw) == None
diff --git a/mapreduce/tests/test_extraction_cdx_grobid.py b/mapreduce/tests/test_extraction_cdx_grobid.py
index 514a830..1d32c9f 100644
--- a/mapreduce/tests/test_extraction_cdx_grobid.py
+++ b/mapreduce/tests/test_extraction_cdx_grobid.py
@@ -121,7 +121,6 @@ def test_parse_cdx_skip(job):
job.mapper_init()
-
print("warc format")
raw = "com,sagepub,cep)/content/28/9/960.full.pdf 20170705062200 http://cep.sagepub.com/content/28/9/960.full.pdf application/pdf 200 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 401 313356621 CITESEERX-CRAWL-2017-06-20-20170705062052659-00043-31209~wbgrp-svc284.us.archive.org~8443.warc.gz"
info, status = job.mapper(None, raw).__next__()