aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--pig/join-cdx-sha1.pig4
1 files changed, 2 insertions, 2 deletions
diff --git a/pig/join-cdx-sha1.pig b/pig/join-cdx-sha1.pig
index 460f8b0..86b9bb6 100644
--- a/pig/join-cdx-sha1.pig
+++ b/pig/join-cdx-sha1.pig
@@ -16,7 +16,7 @@
set mapreduce.job.queuename default
-digests = LOAD '$INPUT_DIGEST' USING PigStorage() AS sha1b32:chararray;
+digests = LOAD '$INPUT_DIGEST' AS sha1b32:chararray;
digests = ORDER digests by sha1b32 ASC PARALLEL 20;
digests = DISTINCT digests;
@@ -29,7 +29,7 @@ cdx = FOREACH cdx GENERATE (chararray)cols.$0 as cdx_surt, (chararray)cols.$1 as
cdx = FILTER cdx BY not cdx_surt matches '-';
cdx = FILTER cdx BY httpstatus matches '200';
cdx = FILTER cdx BY not mimetype matches 'warc/revisit';
-cdx = ORDER cdx by sha1b32 ASC PARALLEL 40;
+cdx = ORDER cdx BY sha1b32 ASC PARALLEL 40;
-- TODO: DISTINCT by (sha1b32, cdx_surt) for efficiency