From 28de71e714c1f5d70adcfd3213dc2433a701a430 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 22 Dec 2019 14:26:04 -0800 Subject: pig: first rev of join-cdx-sha1 script --- pig/join-cdx-sha1.pig | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 pig/join-cdx-sha1.pig (limited to 'pig/join-cdx-sha1.pig') diff --git a/pig/join-cdx-sha1.pig b/pig/join-cdx-sha1.pig new file mode 100644 index 0000000..460f8b0 --- /dev/null +++ b/pig/join-cdx-sha1.pig @@ -0,0 +1,43 @@ + +-- +-- Author: Bryan Newbold +-- Date: December 2020 +-- +-- This pig script is intended to run agains the full (many TByte) GWB CDX, and +-- catch captures that match exact SHA1 (b32 encoded), regardless of mimetype. +-- +-- The process is to filter the CDX for non-revisit HTTP 200s, sort this by +-- SHA1 digest, then join with the (pre-sorted) SHA1 -- b32 input list, and dump +-- output. + +%default INPUT_CDX '' +%default INPUT_DIGEST '' +%default OUTPUT '' + +set mapreduce.job.queuename default + +digests = LOAD '$INPUT_DIGEST' USING PigStorage() AS sha1b32:chararray; +digests = ORDER digests by sha1b32 ASC PARALLEL 20; +digests = DISTINCT digests; + +cdx = LOAD '$INPUT_CDX' AS cdxline:chararray; +cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc'); +cdx = FILTER cdx BY not STARTSWITH (cdxline, ' '); + +cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline; +cdx = FOREACH cdx GENERATE (chararray)cols.$0 as cdx_surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1b32, cdxline; +cdx = FILTER cdx BY not cdx_surt matches '-'; +cdx = FILTER cdx BY httpstatus matches '200'; +cdx = FILTER cdx BY not mimetype matches 'warc/revisit'; +cdx = ORDER cdx by sha1b32 ASC PARALLEL 40; + +-- TODO: DISTINCT by (sha1b32, cdx_surt) for efficiency + +-- Core JOIN +full_join = JOIN cdx BY sha1b32, digests BY sha1b32; + +-- TODO: at most, say 5 CDX lines per sha1b32? + +result = FOREACH full_join GENERATE cdxline; + +STORE result INTO '$OUTPUT' USING PigStorage(); -- cgit v1.2.3