15 files changed, 242 insertions, 29 deletions
diff --git a/pig/README.md b/pig/README.md
index d14d2ae..df8ce68 100644
--- a/pig/README.md
+++ b/pig/README.md
@@ -12,12 +12,13 @@ by `fetch_deps.sh`) due to [dependency/jar issues][pig-bug] in local mode of
 
 To run tests, you need Java installed and `JAVA_HOME` configured.
 
-Fetch dependencies (pig):
+Fetch dependencies (including pig) from top-level directory:
 
-    ./fetch_deps.sh
+    ./fetch_hadoop.sh
 
-Write .pig scripts here, and add a pytho wrapper test to `./tests/` when done.
-Test vector files (input/output) can go in `./tests/files/`.
+Write `.pig` scripts in this directory, and add a python wrapper test to
+`./tests/` when done.  Test vector files (input/output) can go in
+`./tests/files/`.
 
 Run the tests with:
 
diff --git a/pig/fetch_deps.sh b/pig/fetch_deps.sh
deleted file mode 100755
index 4cefa5e..0000000
--- a/pig/fetch_deps.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-#PIG_VERSION="0.12.0-cdh5.2.0"
-# Using more recent version to work around snappy classpath problem
-PIG_VERSION="0.17.0"
-JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
-
-mkdir -p deps/
-cd deps/
-
-# Fetch Pig
-#wget -c https://archive.cloudera.com/cdh5/cdh/5/pig-${PIG_VERSION}.tar.gz
-#wget -c http://mirror.metrocast.net/apache/pig/pig-${PIG_VERSION}/pig-${PIG_VERSION}.tar.gz
-wget -c https://archive.org/serve/hadoop_pig_mirror/pig-${PIG_VERSION}.tar.gz
-tar xvf pig-${PIG_VERSION}.tar.gz
-ln -fs pig-${PIG_VERSION} pig
-./pig/bin/pig -x local -version
-
diff --git a/pig/filter-cdx-paper-pdfs.pig b/pig/filter-cdx-paper-pdfs.pig
index 7e10720..402d340 100644
--- a/pig/filter-cdx-paper-pdfs.pig
+++ b/pig/filter-cdx-paper-pdfs.pig
@@ -30,7 +30,7 @@ cdx = FILTER cdx
         OR surt matches '(?i).+\\).*/(pubs|research|publications?|articles?|proceedings?|papers?|fulltext)/.*'
 
         -- words in domains 
-        OR surt matches '.*(,hal|,eprint|scielo|redalyc|revues|revistas|research|journal).*\\).*'
+        OR surt matches '.*(,hal|,eprint|,ojs|,dspace|scielo|redalyc|revues|revistas|research|journal).*\\).*'
 
         -- DOI-like pattern in URL
         OR surt matches '.*\\).*/10\\.\\d{3,5}/.*';
diff --git a/pig/filter-cdx-pdfs.pig b/pig/filter-cdx-pdfs.pig
new file mode 100644
index 0000000..a2882ac
--- /dev/null
+++ b/pig/filter-cdx-pdfs.pig
@@ -0,0 +1,24 @@
+
+-- Tries to filter down a large CDX file (GWB index) to a subset of PDFs, by mimetype.
+--
+-- Author: Bryan Newbold <bnewbold@archive.org>
+-- Date: May 2018
+
+%default INPUT ''
+%default OUTPUT ''
+
+set mapreduce.job.queuename default
+
+cdx = LOAD '$INPUT' AS cdxline:chararray;
+cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc');
+cdx = FILTER cdx BY not STARTSWITH (cdxline, ' ');
+
+cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline;
+cdx = FOREACH cdx GENERATE (chararray)cols.$0 as url, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, cdxline;
+cdx = FILTER cdx BY not url matches '-';
+cdx = FILTER cdx BY httpstatus matches '200';
+cdx = FILTER cdx BY mimetype matches '.*pdf.*';
+cdx = ORDER cdx by url, timestamp PARALLEL 50;
+cdx = FOREACH cdx GENERATE cdxline;
+STORE cdx INTO '$OUTPUT' USING PigStorage(' ');
+
diff --git a/pig/filter-cdx-ps.pig b/pig/filter-cdx-ps.pig
index 6e80acc..b27a547 100644
--- a/pig/filter-cdx-ps.pig
+++ b/pig/filter-cdx-ps.pig
@@ -1,3 +1,9 @@
+-- Tries to filter down a large CDX file (GWB index) to a subset of postscript
+-- files, by mimetype.
+--
+-- Author: Bryan Newbold <bnewbold@archive.org>
+-- Date: May 2018
+
 %default INPUT ''
 %default OUTPUT ''
 
diff --git a/pig/filter-cdx-source-code-crude.pig b/pig/filter-cdx-source-code-crude.pig
new file mode 100644
index 0000000..589aebd
--- /dev/null
+++ b/pig/filter-cdx-source-code-crude.pig
@@ -0,0 +1,40 @@
+
+-- Tries to filter down a large CDX file (GWB index) to a subset of source code
+-- files by mimetype and file extension.
+-- This is pretty crude and requires the URL to end with the file extension.
+---
+-- Author: Bryan Newbold <bnewbold@archive.org>
+-- Date: October 2019
+
+
+%default INPUT ''
+%default OUTPUT ''
+
+set mapreduce.job.queuename default
+
+cdx = LOAD '$INPUT' AS cdxline:chararray;
+cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc');
+cdx = FILTER cdx BY not STARTSWITH (cdxline, ' ');
+
+cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline;
+cdx = FOREACH cdx GENERATE (chararray)cols.$0 as surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1sum, cdxline;
+cdx = FILTER cdx BY not surt matches '-';
+cdx = FILTER cdx BY httpstatus matches '200';
+cdx = FILTER cdx BY mimetype matches '.*text.*';
+
+-- This is the core regex
+cdx = FILTER cdx
+
+        -- file suffix
+        BY surt matches '.*\\).*\\.(c|h|py|java)';
+
+-- DISTINCT by sha1 column
+cdx_uniq = FOREACH (GROUP cdx BY sha1sum) {
+    r = TOP(1, 0, $1);
+    GENERATE FLATTEN(r);
+};
+
+cdx_uniq = ORDER cdx_uniq by surt, timestamp PARALLEL 50;
+cdx_uniq = FOREACH cdx_uniq GENERATE cdxline;
+STORE cdx_uniq INTO '$OUTPUT' USING PigStorage(' ');
+
diff --git a/pig/filter-cdx-tarball.pig b/pig/filter-cdx-tarball.pig
new file mode 100644
index 0000000..d0be0f7
--- /dev/null
+++ b/pig/filter-cdx-tarball.pig
@@ -0,0 +1,38 @@
+
+-- Tries to filter down a large CDX file (GWB index) to a subset of tarballs
+-- (.tar.gz). Intention is to find software code that isn't in, eg, git.
+--
+-- Author: Bryan Newbold <bnewbold@archive.org>
+-- Date: May 2018
+
+
+%default INPUT ''
+%default OUTPUT ''
+
+set mapreduce.job.queuename default
+
+cdx = LOAD '$INPUT' AS cdxline:chararray;
+cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc');
+cdx = FILTER cdx BY not STARTSWITH (cdxline, ' ');
+
+cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline;
+cdx = FOREACH cdx GENERATE (chararray)cols.$0 as surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1sum, cdxline;
+cdx = FILTER cdx BY not surt matches '-';
+cdx = FILTER cdx BY httpstatus matches '200';
+cdx = FILTER cdx BY mimetype matches '.*(octet|gzip|gtar|tgz).*';
+
+-- This is the core regex
+cdx = FILTER cdx
+        -- .tar.gz in URL
+        BY surt matches '(?i).+\\).*\\.tar\\.gz.*';
+
+-- DISTINCT by sha1 column
+cdx_uniq = FOREACH (GROUP cdx BY sha1sum) {
+    r = TOP(1, 0, $1);
+    GENERATE FLATTEN(r);
+};
+
+cdx_uniq = ORDER cdx_uniq by surt, timestamp PARALLEL 50;
+cdx_uniq = FOREACH cdx_uniq GENERATE cdxline;
+STORE cdx_uniq INTO '$OUTPUT' USING PigStorage(' ');
+
diff --git a/pig/join-cdx-sha1.pig b/pig/join-cdx-sha1.pig
new file mode 100644
index 0000000..86b9bb6
--- /dev/null
+++ b/pig/join-cdx-sha1.pig
@@ -0,0 +1,43 @@
+
+--
+-- Author: Bryan Newbold <bnewbold@archive.org>
+-- Date: December 2020
+--
+-- This pig script is intended to run agains the full (many TByte) GWB CDX, and
+-- catch captures that match exact SHA1 (b32 encoded), regardless of mimetype.
+--
+-- The process is to filter the CDX for non-revisit HTTP 200s, sort this by
+-- SHA1 digest, then join with the (pre-sorted) SHA1 -- b32 input list, and dump
+-- output.
+
+%default INPUT_CDX ''
+%default INPUT_DIGEST ''
+%default OUTPUT ''
+
+set mapreduce.job.queuename default
+
+digests = LOAD '$INPUT_DIGEST' AS sha1b32:chararray;
+digests = ORDER digests by sha1b32 ASC PARALLEL 20;
+digests = DISTINCT digests;
+
+cdx = LOAD '$INPUT_CDX' AS cdxline:chararray;
+cdx = FILTER cdx BY not STARTSWITH (cdxline, 'filedesc');
+cdx = FILTER cdx BY not STARTSWITH (cdxline, ' ');
+
+cdx = FOREACH cdx GENERATE STRSPLIT(cdxline,'\\s+') as cols, cdxline;
+cdx = FOREACH cdx GENERATE (chararray)cols.$0 as cdx_surt, (chararray)cols.$1 as timestamp, (chararray)cols.$3 as mimetype, (chararray)cols.$4 as httpstatus, (chararray)cols.$5 as sha1b32, cdxline;
+cdx = FILTER cdx BY not cdx_surt matches '-';
+cdx = FILTER cdx BY httpstatus matches '200';
+cdx = FILTER cdx BY not mimetype matches 'warc/revisit';
+cdx = ORDER cdx BY sha1b32 ASC PARALLEL 40;
+
+-- TODO: DISTINCT by (sha1b32, cdx_surt) for efficiency
+
+-- Core JOIN
+full_join = JOIN cdx BY sha1b32, digests BY sha1b32;
+
+-- TODO: at most, say 5 CDX lines per sha1b32?
+
+result = FOREACH full_join GENERATE cdxline;
+
+STORE result INTO '$OUTPUT' USING PigStorage();
diff --git a/pig/tests/files/example.sha1b32 b/pig/tests/files/example.sha1b32
new file mode 100644
index 0000000..20a1357
--- /dev/null
+++ b/pig/tests/files/example.sha1b32
@@ -0,0 +1,4 @@
+EJWYVOPONJRARK7SGG6COFRN7CSTHROY
+V32E3CCO7NMI2M4OHLKG73DXD72LR4B2
+3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ
+E3WSNQ7JAFOW7N3ZJ6GLV27T52T25JDK
diff --git a/pig/tests/files/sourcecode.cdx b/pig/tests/files/sourcecode.cdx
new file mode 100644
index 0000000..eeb397c
--- /dev/null
+++ b/pig/tests/files/sourcecode.cdx
@@ -0,0 +1,6 @@
+# match
+edu,cmu,cs,adm,reports-archive)/anon/usr0/ftp/usr0/anon/2002/cmu-cs-02-119.java 20170706005950 http://reports-archive.adm.cs.cmu.edu/anon/usr0/ftp/usr0/anon/2002/CMU-CS-02-119.java text/plain 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 361006 17120058 CITESEERX-CRAWL-2017-06-20-20170706004100259-00924-00932-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170706005946792-00926-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+# no
+fi,tkk,lib)/diss/2001/isbn951225459x/isbn951225459x.pyc 20170705074926 http://lib.tkk.fi/Diss/2001/isbn951225459X/isbn951225459X.pyc text/plain 200 KJBCOT7LGBNIAVGEGPUELK5OK6RTFORR - - 344175 255650124 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705074843696-00134-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+# no
+org,oxfordjournals,nar)/cgi/reprint/gkl1060v1.pdf 20170706035441 http://nar.oxfordjournals.org/cgi/reprint/gkl1060v1.pdf text/html 301 OX6MLVDFURLT2KSYCXUYW2PZNOVFSEVF - - 697 49346051 CITESEERX-CRAWL-2017-06-20-20170706034741172-00140-00149-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706035435634-00148-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
diff --git a/pig/tests/files/tarballs.cdx b/pig/tests/files/tarballs.cdx
new file mode 100644
index 0000000..7a81b79
--- /dev/null
+++ b/pig/tests/files/tarballs.cdx
@@ -0,0 +1,10 @@
+#http://research.fit.edu/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf
+#http://ijs.sgmjournals.org:80/cgi/reprint/54/6/2217.pdf
+#http://eprints.ecs.soton.ac.uk/12020/1/mind-the-semantic-gap.pdf
+#http://eprint.uq.edu.au/archive/00004120/01/R103_Forrester_pp.pdf
+
+# should match 2:
+
+edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.pdf 20170706005950 http://mit.edu/file.pdf application/pdf 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz
+edu,fit,research)/sealevelriselibrary/documents/doc_mgr/448/Florida_Keys_Low_Island_Biodiversity_&_SLR_-_Ross_et_al_2009.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/octet-stream 200 NQHD36X5MNZPWFNMD5LFOYZSFGCHUN3I - - 123 456 CRAWL/CRAWL.warc.gz
+org,sgmjournals,ijs)//cgi/reprint/54/6/2217.tar.gz 20170706005950 http://mit.edu/file.tar.gz application/gzip 200 TQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 123 456 CRAWL/CRAWL.warc.gz
diff --git a/pig/tests/pighelper.py b/pig/tests/pighelper.py
index 4aa4259..95e0426 100644
--- a/pig/tests/pighelper.py
+++ b/pig/tests/pighelper.py
@@ -17,6 +17,9 @@ import unittest
 import subprocess
 
 
+def count_lines(s):
+    return len([l for l in s.strip().split('\n') if len(l) > 0])
+
 class PigTestHelper(unittest.TestCase):
 
     @classmethod
@@ -50,7 +53,7 @@ class PigTestHelper(unittest.TestCase):
         return retval
 
     def run_pig(self, script_path, in_file, **kwargs):
-        """Convenience helper around run_pig().
+        """Convenience helper around run_pig_raw().
         
         INPUT parameter is set to in_file.
         OUTPUT parameter is set to a random file.
diff --git a/pig/tests/test_filter_cdx_paper_pdfs.py b/pig/tests/test_filter_cdx_paper_pdfs.py
index a8ebd9f..c2d2e6b 100644
--- a/pig/tests/test_filter_cdx_paper_pdfs.py
+++ b/pig/tests/test_filter_cdx_paper_pdfs.py
@@ -1,10 +1,8 @@
 
 import os
 import unittest
-from pighelper import PigTestHelper
+from pighelper import PigTestHelper, count_lines
 
-def count_lines(s):
-    return len([l for l in s.strip().split('\n') if len(l) > 0])
 
 class TestFilterCDXPaperPdfs(PigTestHelper):
 
diff --git a/pig/tests/test_filter_software.py b/pig/tests/test_filter_software.py
new file mode 100644
index 0000000..cce90b4
--- /dev/null
+++ b/pig/tests/test_filter_software.py
@@ -0,0 +1,16 @@
+
+import os
+import unittest
+from pighelper import PigTestHelper, count_lines
+
+
+class TestFilterCDXSoftware(PigTestHelper):
+
+    def test_tarballs(self):
+        r = self.run_pig("filter-cdx-tarball.pig", "tests/files/tarballs.cdx")
+        assert count_lines(r) == 2
+
+    def test_source_code(self):
+        r = self.run_pig("filter-cdx-source-code-crude.pig", "tests/files/sourcecode.cdx")
+        assert count_lines(r) == 1
+
diff --git a/pig/tests/test_join_cdx.py b/pig/tests/test_join_cdx.py
new file mode 100644
index 0000000..e6eca6a
--- /dev/null
+++ b/pig/tests/test_join_cdx.py
@@ -0,0 +1,44 @@
+
+import os
+import unittest
+import tempfile
+import subprocess
+from pighelper import PigTestHelper, count_lines
+
+class TestJoinCDXSha1(PigTestHelper):
+
+    def run_pig_join(self, script_path, cdx_file, digest_file, **kwargs):
+        """Convenience helper around run_pig().
+        
+        INPUT parameter is set to in_file.
+        OUTPUT parameter is set to a random file.
+        Any keyword args are passed as parameters.
+        """
+
+        pargs = []
+        for key, value in kwargs.items():
+            pargs.append('-p')
+            pargs.append('{}={}'.format(key, value))
+
+        out_file = tempfile.mktemp(dir=self._tmpdir)
+        params = [
+            '-f', script_path,
+            '-p', 'INPUT_CDX={}'.format(cdx_file),
+            '-p', 'INPUT_DIGEST={}'.format(digest_file),
+            '-p', 'OUTPUT={}'.format(out_file),
+            ] + pargs
+        status = self.run_pig_raw(params)
+        assert status.returncode == 0
+        # Capture all the part-r-* files
+        print("out_file: {}".format(out_file))
+        subprocess.run("/bin/ls -la {}/part-*".format(out_file), shell=True)
+        sub = subprocess.run("/bin/cat {}/part-*".format(out_file), stdout=subprocess.PIPE, shell=True)
+        out = sub.stdout.decode('utf-8')
+        print(out)
+        return out
+
+    # TODO: helper to verify that output matches an expected file
+
+    def test_thing(self):
+        r = self.run_pig_join("join-cdx-sha1.pig", "tests/files/example.cdx", "tests/files/example.sha1b32")
+        assert count_lines(r) == 4