From 2a1c887309305187d785b34a16c1868d26cb3273 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 7 May 2018 23:41:10 -0700 Subject: WIP on filter-cdx-join-urls.pig --- pig/filter-cdx-join-urls.pig | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 pig/filter-cdx-join-urls.pig (limited to 'pig') diff --git a/pig/filter-cdx-join-urls.pig b/pig/filter-cdx-join-urls.pig new file mode 100644 index 0000000..b396c82 --- /dev/null +++ b/pig/filter-cdx-join-urls.pig @@ -0,0 +1,37 @@ + +-- +-- Author: Bryan Newbold +-- Date: May 2018 + +%default INPUT_CDX '' +%default INPUT_URLS '' +%default OUTPUT '' + +REGISTER /home/webcrawl/pig-scripts/jars/ia-web-commons-jar-with-dependencies-CDH3.jar; +REGISTER /home/webcrawl/pig-scripts/jars/pigtools.jar; +DEFINE SURTURL pigtools.SurtUrlKey(); + +set mapreduce.job.queuename default + +urls = LOAD '$INPUT_URLS' USING PigStorage() AS url:chararray; +surts = FOREACH urls GENERATE SURTURL(url) AS url_surt; +surts = ORDER surts by url_surt ASC PARALLEL 10; +surts = DISTINCT surts; + +cdx = LOAD '$INPUT' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as cdx_surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, cdxline; +cdx = FILTER cdx BY not cdx_surt matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY mimetype matches '.*pdf.*'; + +-- Core JOIN +full_join = JOIN cdx BY cdx_surt, surts BY url_surt; + +result = FOREACH full_join GENERATE cdxline; +result = DISTINCT result; + +STORE result INTO '$OUTPUT' USING PigStorage(); -- cgit v1.2.3