From 54dabe601eaa19d0495d9a102b34e9daa056457d Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Thu, 17 Oct 2019 17:19:34 +0100 Subject: new/additional GWB CDX filter scripts --- pig/filter-cdx-pdfs.pig | 24 ++++++++++++++++++++++ pig/filter-cdx-ps.pig | 6 ++++++ pig/filter-cdx-source-code-crude.pig | 40 ++++++++++++++++++++++++++++++++++++ pig/filter-cdx-tarball.pig | 38 ++++++++++++++++++++++++++++++++++ pig/tests/files/sourcecode.cdx | 6 ++++++ pig/tests/files/tarballs.cdx | 10 +++++++++ pig/tests/test_filter_software.py | 18 ++++++++++++++++ 7 files changed, 142 insertions(+) create mode 100644 pig/filter-cdx-pdfs.pig create mode 100644 pig/filter-cdx-source-code-crude.pig create mode 100644 pig/filter-cdx-tarball.pig create mode 100644 pig/tests/files/sourcecode.cdx create mode 100644 pig/tests/files/tarballs.cdx create mode 100644 pig/tests/test_filter_software.py (limited to 'pig') diff --git a/pig/filter-cdx-pdfs.pig b/pig/filter-cdx-pdfs.pig new file mode 100644 index 0000000..a2882ac --- /dev/null +++ b/pig/filter-cdx-pdfs.pig @@ -0,0 +1,24 @@ + +-- Tries to filter down a large CDX file (GWB index) to a subset of PDFs, by mimetype. +-- +-- Author: Bryan Newbold +-- Date: May 2018 + +%default INPUT '' +%default OUTPUT '' + +set mapreduce.job.queuename default + +cdx = LOAD '$INPUT' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as url, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, cdxline; +cdx = FILTER cdx BY not url matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY mimetype matches '.*pdf.*'; +cdx = ORDER cdx by url, timestamp PARALLEL 50; +cdx = FOREACH cdx GENERATE cdxline; +STORE cdx INTO '$OUTPUT' USING PigStorage(' '); + diff --git a/pig/filter-cdx-ps.pig b/pig/filter-cdx-ps.pig index 6e80acc..b27a547 100644 --- a/pig/filter-cdx-ps.pig +++ b/pig/filter-cdx-ps.pig @@ -1,3 +1,9 @@ +-- Tries to filter down a large CDX file (GWB index) to a subset of postscript +-- files, by mimetype. +-- +-- Author: Bryan Newbold +-- Date: May 2018 + %default INPUT '' %default OUTPUT '' diff --git a/pig/filter-cdx-source-code-crude.pig b/pig/filter-cdx-source-code-crude.pig new file mode 100644 index 0000000..589aebd --- /dev/null +++ b/pig/filter-cdx-source-code-crude.pig @@ -0,0 +1,40 @@ + +-- Tries to filter down a large CDX file (GWB index) to a subset of source code +-- files by mimetype and file extension. +-- This is pretty crude and requires the URL to end with the file extension. +--- +-- Author: Bryan Newbold +-- Date: October 2019 + + +%default INPUT '' +%default OUTPUT '' + +set mapreduce.job.queuename default + +cdx = LOAD '$INPUT' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1sum, cdxline; +cdx = FILTER cdx BY not surt matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY mimetype matches '.*text.*'; + +-- This is the core regex +cdx = FILTER cdx + + -- file suffix + BY surt matches '.*\\).*\\.(c|h|py|java)'; + +-- DISTINCT by sha1 column +cdx_uniq = FOREACH (GROUP cdx BY sha1sum) { + r = TOP(1, 0, $1); + GENERATE FLATTEN(r); +}; + +cdx_uniq = ORDER cdx_uniq by surt, timestamp PARALLEL 50; +cdx_uniq = FOREACH cdx_uniq GENERATE cdxline; +STORE cdx_uniq INTO '$OUTPUT' USING PigStorage(' '); + diff --git a/pig/filter-cdx-tarball.pig b/pig/filter-cdx-tarball.pig new file mode 100644 index 0000000..d0be0f7 --- /dev/null +++ b/pig/filter-cdx-tarball.pig @@ -0,0 +1,38 @@ + +-- Tries to filter down a large CDX file (GWB index) to a subset of tarballs +-- (.tar.gz). Intention is to find software code that isn't in, eg, git. +-- +-- Author: Bryan Newbold +-- Date: May 2018 + + +%default INPUT '' +%default OUTPUT '' + +set mapreduce.job.queuename default + +cdx = LOAD '$INPUT' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1sum, cdxline; +cdx = FILTER cdx BY not surt matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY mimetype matches '.*(octet|gzip|gtar|tgz).*'; + +-- This is the core regex +cdx = FILTER cdx + -- .tar.gz in URL + BY surt matches '(?i).+\\).*\\.tar\\.gz.*'; + +-- DISTINCT by sha1 column +cdx_uniq = FOREACH (GROUP cdx BY sha1sum) { + r = TOP(1, 0, $1); + GENERATE FLATTEN(r); +}; + +cdx_uniq = ORDER cdx_uniq by surt, timestamp PARALLEL 50; +cdx_uniq = FOREACH cdx_uniq GENERATE cdxline; +STORE cdx_uniq INTO '$OUTPUT' USING PigStorage(' '); + diff --git a/pig/tests/files/sourcecode.cdx b/pig/tests/files/sourcecode.cdx new file mode 100644 index 0000000..eeb397c --- /dev/null +++ b/pig/tests/files/sourcecode.cdx @@ -0,0 +1,6 @@ +# match +edu,cmu,cs,adm,reports-archive)/anon/usr0/ftp/usr0/anon/2002/cmu-cs-02-119.java 20170706005950 http://reports-archive.adm.cs.cmu.edu/anon/usr0/ftp/usr0/anon/2002/CMU-CS-02-119.java text/plain 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 361006 17120058 CITESEERX-CRAWL-2017-06-20-20170706004100259-00924-00932-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170706005946792-00926-31209~wbgrp-svc284.us.archive.org~8443.warc.gz +# no +fi,tkk,lib)/diss/2001/isbn951225459x/isbn951225459x.pyc 20170705074926 http://lib.tkk.fi/Diss/2001/isbn951225459X/isbn951225459X.pyc text/plain 200 KJBCOT7LGBNIAVGEGPUELK5OK6RTFORR - - 344175 255650124 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705074843696-00134-31209~wbgrp-svc284.us.archive.org~8443.warc.gz +# no +org,oxfordjournals,nar)/cgi/reprint/gkl1060v1.pdf 20170706035441 http://nar.oxfordjournals.org/cgi/reprint/gkl1060v1.pdf text/html 301 OX6MLVDFURLT2KSYCXUYW2PZNOVFSEVF - - 697 49346051 CITESEERX-CRAWL-2017-06-20-20170706034741172-00140-00149-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706035435634-00148-3671~wbgrp-svc285.us.archive.org~8443.warc.gz diff --git a/pig/tests/files/tarballs.cdx b/pig/tests/files/tarballs.cdx new file mode 100644 index 0000000..7a81b79 --- /dev/null +++ b/pig/tests/files/tarballs.cdx @@ -0,0 +1,10 @@ +#http://research.fit.edu/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf +#http://ijs.sgmjournals.org:80/cgi/reprint/54/6/2217.pdf +#http://eprints.ecs.soton.ac.uk/12020/1/mind-the-semantic-gap.pdf +#http://eprint.uq.edu.au/archive/00004120/01/R103_Forrester_pp.pdf + +# should match 2: + +edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf 20170706005950 http://mit.edu/file.pdf application/pdf 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz +edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/octet-stream 200 NQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz +org,sgmjournals,ijs)//cgi/reprint/54/6/2217.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/gzip 200 TQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 123 456 CRAWL/CRAWL.warc.gz diff --git a/pig/tests/test_filter_software.py b/pig/tests/test_filter_software.py new file mode 100644 index 0000000..f0ea1b6 --- /dev/null +++ b/pig/tests/test_filter_software.py @@ -0,0 +1,18 @@ + +import os +import unittest +from pighelper import PigTestHelper + +def count_lines(s): + return len([l for l in s.strip().split('\n') if len(l) > 0]) + +class TestFilterCDXSoftware(PigTestHelper): + + def test_tarballs(self): + r = self.run_pig("filter-cdx-tarball.pig", "tests/files/tarballs.cdx") + assert count_lines(r) == 2 + + def test_source_code(self): + r = self.run_pig("filter-cdx-source-code-crude.pig", "tests/files/sourcecode.cdx") + assert count_lines(r) == 1 + -- cgit v1.2.3