diff options
author | Martin Czygan <martin.czygan@gmail.com> | 2021-04-09 23:57:57 +0200 |
---|---|---|
committer | Martin Czygan <martin.czygan@gmail.com> | 2021-04-19 20:29:17 +0200 |
commit | ef8ccdc83e2817a85befab229e11b5a4e34b302e (patch) | |
tree | 76db5d117dc1c7f841f3be80a03d3f71d0316331 /pig/filter-cdx-join-urls.pig | |
parent | c0e097776dd191ee2f5321239b70a0fb7e9b08ef (diff) | |
download | refcat-ef8ccdc83e2817a85befab229e11b5a4e34b302e.tar.gz refcat-ef8ccdc83e2817a85befab229e11b5a4e34b302e.zip |
pig stubs
Diffstat (limited to 'pig/filter-cdx-join-urls.pig')
-rw-r--r-- | pig/filter-cdx-join-urls.pig | 43 |
1 files changed, 43 insertions, 0 deletions
diff --git a/pig/filter-cdx-join-urls.pig b/pig/filter-cdx-join-urls.pig new file mode 100644 index 0000000..3d06804 --- /dev/null +++ b/pig/filter-cdx-join-urls.pig @@ -0,0 +1,43 @@ + +-- +-- Author: Bryan Newbold <bnewbold@archive.org> +-- Date: May 2018 + +%default INPUT_CDX '' +%default INPUT_URLS '' +%default OUTPUT '' + +REGISTER /home/webcrawl/pig-scripts/jars/ia-web-commons-jar-with-dependencies-CDH3.jar; +REGISTER /home/webcrawl/pig-scripts/jars/pigtools.jar; +DEFINE SURTURL pigtools.SurtUrlKey(); + +set mapreduce.job.queuename default + +urls = LOAD '$INPUT_URLS' USING PigStorage() AS url:chararray; +surts = FOREACH urls GENERATE SURTURL(url) AS url_surt; +surts = ORDER surts by url_surt ASC PARALLEL 10; +surts = DISTINCT surts; + +cdx = LOAD '$INPUT_CDX' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as cdx_surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1sum, cdxline; +cdx = FILTER cdx BY not cdx_surt matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY mimetype matches '.*pdf.*'; + +-- Core JOIN +full_join = JOIN cdx BY cdx_surt, surts BY url_surt; + +-- DISTINCT by sha1 column +full_uniq = FOREACH (GROUP full_join BY sha1sum) { + r = TOP(1, 0, $1); + GENERATE FLATTEN(r); +}; + +result = FOREACH full_uniq GENERATE cdxline; +result = DISTINCT result; + +STORE result INTO '$OUTPUT' USING PigStorage(); |