progress on pig tests

8 files changed, 127 insertions, 10 deletions

diff --git a/pig/.gitignore b/pig/.gitignore
new file mode 100644
index 0000000..504f811
--- /dev/null
+++ b/pig/.gitignore
@@ -0,0 +1,2 @@
+deps
+*.log
diff --git a/pig/Pipfile b/pig/Pipfile
index af1a0e0..dbdef21 100644
--- a/pig/Pipfile
+++ b/pig/Pipfile
@@ -17,4 +17,4 @@ nose = "*"
 
 [requires]
 
-python_version = "2.7"
+python_version = "3.5"
diff --git a/pig/Pipfile.lock b/pig/Pipfile.lock
index 2ae4d69..3ac834f 100644
--- a/pig/Pipfile.lock
+++ b/pig/Pipfile.lock
@@ -1,24 +1,24 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "8a7f3e832d1c7a39918cabd60145d566e66b48ca8f7ada59b0128a28a8096398"
+            "sha256": "6591f4ebe3f7ad124ef9bae389cd9c11b2fe1d6936bf65c4cec64cb016ef0661"
         },
         "host-environment-markers": {
             "implementation_name": "cpython",
-            "implementation_version": "0",
+            "implementation_version": "3.5.3",
             "os_name": "posix",
             "platform_machine": "x86_64",
             "platform_python_implementation": "CPython",
             "platform_release": "4.9.0-6-amd64",
             "platform_system": "Linux",
             "platform_version": "#1 SMP Debian 4.9.82-1+deb9u3 (2018-03-02)",
-            "python_full_version": "2.7.13",
-            "python_version": "2.7",
-            "sys_platform": "linux2"
+            "python_full_version": "3.5.3",
+            "python_version": "3.5",
+            "sys_platform": "linux"
         },
         "pipfile-spec": 6,
         "requires": {
-            "python_version": "2.7"
+            "python_version": "3.5"
         },
         "sources": [
             {
diff --git a/pig/README.md b/pig/README.md
index e47e31d..c518591 100644
--- a/pig/README.md
+++ b/pig/README.md
@@ -13,7 +13,7 @@ https://hub.docker.com/r/chalimartines/local-pig
     ln -s pig-0.12.0-cdh5.0.1/pig-0.12.0-cdh5.0.1.jar pig-0.12.0-cdh5.0.1/pig.jar
     ./pig-*/bin/pig -x local -version
 
-    #XXX:
+    #XXX: don't need Hadoop?
     #wget https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.3.0-cdh5.0.1.tar.gz
     #tar xvf hadoop-*.tar.gz
     #export HADOOP_HOME=hadoop-2.3*
diff --git a/pig/fetch_deps.sh b/pig/fetch_deps.sh
new file mode 100755
index 0000000..529f8f7
--- /dev/null
+++ b/pig/fetch_deps.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# If you change this, also update tests/pighelper.py
+PIG_VERSION="0.12.0-cdh5.0.1"
+
+mkdir -p deps/
+cd deps/
+wget -c https://archive.cloudera.com/cdh5/cdh/5/pig-${PIG_VERSION}.tar.gz
+tar xvf pig-${PIG_VERSION}.tar.gz
+ln -fs pig-${PIG_VERSION} pig
+cd pig
+ln -fs pig-${PIG_VERSION}.jar pig.jar
+cd ..
+
+JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
+./pig/bin/pig -x local -version
+
diff --git a/pig/tests/files/example.cdx b/pig/tests/files/example.cdx
new file mode 100644
index 0000000..84e3271
--- /dev/null
+++ b/pig/tests/files/example.cdx
@@ -0,0 +1,20 @@
+edu,cmu,cs,adm,reports-archive)/anon/usr0/ftp/usr0/anon/2002/cmu-cs-02-119.pdf 20170706005950 http://reports-archive.adm.cs.cmu.edu/anon/usr0/ftp/usr0/anon/2002/CMU-CS-02-119.pdf application/pdf 200 MQHD36X5MNZPWFNMD5LFOYZSFGCHUN3V - - 361006 17120058 CITESEERX-CRAWL-2017-06-20-20170706004100259-00924-00932-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170706005946792-00926-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+fi,tkk,lib)/diss/2001/isbn951225459x/isbn951225459x.pdf 20170705074926 http://lib.tkk.fi/Diss/2001/isbn951225459X/isbn951225459X.pdf application/pdf 200 KJBCOT7LGBNIAVGEGPUELK5OK6RTFORR - - 344175 255650124 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705074843696-00134-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+org,oxfordjournals,nar)/cgi/reprint/gkl1060v1.pdf 20170706035441 http://nar.oxfordjournals.org/cgi/reprint/gkl1060v1.pdf text/html 301 OX6MLVDFURLT2KSYCXUYW2PZNOVFSEVF - - 697 49346051 CITESEERX-CRAWL-2017-06-20-20170706034741172-00140-00149-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706035435634-00148-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+org,ifaamas)/proceedings/aamas09/pdf/01_full%20papers/02_08_fp_0272.pdf 20170706081902 http://www.ifaamas.org/Proceedings/aamas09/pdf/01_Full%20Papers/02_08_FP_0272.pdf application/pdf 200 GYHX35QJWRJELWJ5GDQZPTPOUUZOCTKF - - 251180 34635154 CITESEERX-CRAWL-2017-06-20-20170706081825105-00419-00428-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706081838210-00420-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+de,fau,cs)/publications/2014/lukas_14_masterthesis.pdf 20170705101722 http://www4.cs.fau.de/Publications/2014/lukas_14_masterthesis.pdf application/pdf 200 GIUQT7SXZ33TWEFBM2MWURJI2M3QE3IW - - 1290532 71068435 CITESEERX-CRAWL-2017-06-20-20170705101605019-00279-00288-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705101714659-00281-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+de,bund,jki,pub)/index.php/jabfq/article/download/3568/4462 20170706041152 http://pub.jki.bund.de/index.php/JABFQ/article/download/3568/4462/ text/html 301 XZBNO24W2ZPQQMJYE6YUUCSRUF7G3ZBT - - 552 417292708 CITESEERX-CRAWL-2017-06-20-20170706040506112-00160-00169-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706041021844-00165-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+whois://whois.arin.net/z+%2B+132.177.133.114 20170713120653 whois://whois.arin.net/z+%2B+132.177.133.114 text/plain - IDEID4YQ6MVJSOE57NPVDLL53ZB3J4DX - - 876 30983517 CITESEERX-CRAWL-2017-06-20-20170707064626094-01007-01015-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170711214025652-01014-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+za,co,csir,researchspace)/dspace/bitstream/10204/4048/1/smith2_2010.pdf 20170706094159 http://researchspace.csir.co.za/dspace/bitstream/10204/4048/1/Smith2_2010.pdf unk 301 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 446 104830407 CITESEERX-CRAWL-2017-06-20-20170706093829986-00509-00518-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706094137978-00512-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+org,annals)/article.aspx?articleid=705034 20170707013120 http://annals.org/article.aspx?articleid=705034 text/html 301 QQYKL57QSERLFM3LXSWMNOFXMOCN7C5G - - 22665 28113974 CITESEERX-CRAWL-2017-06-20-20170707013100780-00967-00976-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170707013100780-00967-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+org,annals)/pdfaccess.ashx?url=/data/journals/aim/20105/0000605-200512200-00013.pdf 20170707045304 http://annals.org/pdfaccess.ashx?url=/data/journals/aim/20105/0000605-200512200-00013.pdf text/html 302 423S7EMGLCVIZ3FLVD7TLAG75HWE4RGI - - 644 222908628 CITESEERX-CRAWL-2017-06-20-20170707042504366-00997-01006-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170707045044604-00999-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+com,sagepub,spi)/content/28/4/501.full.pdf 20170705092027 http://spi.sagepub.com/content/28/4/501.full.pdf unk 301 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 396 553180242 CITESEERX-CRAWL-2017-06-20-20170705091311851-00219-00228-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705091759818-00223-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+ir,mediaj)/favicon.ico 20170705075240 http://mediaj.ir/favicon.ico text/html 404 E3WSNQ7JAFOW7N3ZJ6GLV27T52T25JDK - - 589 455827180 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705075051100-00135-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+com,sagepub,jpr)/content/8/3-4/239.full.pdf 20170705074931 http://jpr.sagepub.com/content/8/3-4/239.full.pdf unk 301 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 400 270368088 CITESEERX-CRAWL-2017-06-20-20170705074433815-00129-00138-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705074843696-00134-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+jp,co,nittuden)/business/pdf/transparent_thermoplastic_resin_with_electron_beam_cross-linking.pdf 20170706083459 http://www.nittuden.co.jp/business/pdf/Transparent_Thermoplastic_Resin_with_Electron_Beam_Cross-Linking.pdf application/pdf 200 V32E3CCO7NMI2M4OHLKG73DXD72LR4B2 - - 715081 761088410 CITESEERX-CRAWL-2017-06-20-20170706082646066-00429-00438-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706083257353-00436-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+lt,lms)/robots.txt 20170705122708 http://www.lms.lt/robots.txt text/plain 200 PF3HTQQT2ULYRWFLJGUWZKHTVZUVMZ2F - - 592 668333707 CITESEERX-CRAWL-2017-06-20-20170705121748408-00399-00408-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705122352502-00406-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+hu,bme,phy)/~szalay/pub/multipartcriteriaposter.pdf 20170705124828 http://www.phy.bme.hu/%7Eszalay/pub/multipartcriteriaPoster.pdf application/pdf 200 L3TUEEZLBJTHAVH74B5N426FAIDBCCOE - - 187866 964760782 CITESEERX-CRAWL-2017-06-20-20170705123641979-00419-00428-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705124315591-00426-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+org,adb,openaccess)/bitstream/handle/11540/1260/new-regime-sme-finance-asia.pdf;jsessionid=f966a3bdac9882ec5a7c326b130f6f81?sequence=1 20170705090940 https://openaccess.adb.org/bitstream/handle/11540/1260/new-regime-sme-finance-asia.pdf%3Bjsessionid%3DF966A3BDAC9882EC5A7C326B130F6F81?sequence%3D1 unk 301 3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ - - 515 634039376 CITESEERX-CRAWL-2017-06-20-20170705090333400-00209-00218-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705090728803-00212-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+org,physiology,ajpregu)/content/272/4/r1084 20170706131006 http://ajpregu.physiology.org/content/272/4/R1084 text/html 200 3FOQSKT4WBYOUA6VKKJCEQCN6QF35ANT - - 27346 336293585 CITESEERX-CRAWL-2017-06-20-20170706130432396-00707-00716-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706130850866-00711-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
+de,desy,www-it)/common/documentation/cd-docs/sc2002/paperpdf/pap234.pdf 20170705121813 http://www-it.desy.de/common/documentation/cd-docs/SC2002/paperpdf/pap234.pdf application/pdf 200 BONCZ4NNGRNYR22ASFVU7VYTQ24RRNP4 - - 72421 381715704 CITESEERX-CRAWL-2017-06-20-20170705120827801-00389-00398-wbgrp-svc284/CITESEERX-CRAWL-2017-06-20-20170705121708700-00397-31209~wbgrp-svc284.us.archive.org~8443.warc.gz
+org,oxfordjournals,bmb)/content/28/3/247.full.pdf 20170706014948 http://bmb.oxfordjournals.org/content/28/3/247.full.pdf text/html 301 EJWYVOPONJRARK7SGG6COFRN7CSTHROY - - 643 119398161 CITESEERX-CRAWL-2017-06-20-20170706014800946-00020-00029-wbgrp-svc285/CITESEERX-CRAWL-2017-06-20-20170706014907678-00022-3671~wbgrp-svc285.us.archive.org~8443.warc.gz
diff --git a/pig/tests/pighelper.py b/pig/tests/pighelper.py
new file mode 100644
index 0000000..cee074d
--- /dev/null
+++ b/pig/tests/pighelper.py
@@ -0,0 +1,69 @@
+"""
+A helper class for locally testing Pig scripts.
+
+author: Bryan Newbold <bnewbold@archive.org>
+"""
+import os
+import tempfile
+import unittest
+import subprocess
+from nose.tools import *
+
+
+class PigTestHelper(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+
+        cls._pigpath= "./deps/pig/bin/pig"
+        cls._base = [cls._pigpath,
+            '-x', 'local',
+            '-log4jconf', 'pig_log4j.properties',
+            '-stop_on_failure']
+
+        # Check that pig is functioning
+        if subprocess.call(cls._base + ['-version']) != 0:
+            raise unittest.SkipTest("Failed to find and run Pig")
+
+        # Classpath?
+        # os.path.join("pig-0.12.0-cdh5.0.1", "pig.jar"),
+        # os.path.join("pig-0.12.0-cdh5.0.1", "lib", "*"),
+        # "hadoop-2.3.0-cdh5.0.1"
+
+    def setUp(self):
+        self._tmpdir = tempfile.mkdtemp()
+
+    def tearDown(self):
+        os.rmdir(self._tmpdir)
+
+    def run_pig_raw(self, params):
+        """Low-level variant with params appended directly. Returns
+        CompletedProcess, raises an error if return value isn't succes"""
+
+        retval = subprocess.run(self._base + params,
+            timeout=20.0,
+            check=True)
+        return retval
+
+    def run_pig(self, script_path, in_file, **kwargs):
+        """Convenience helper around run_pig().
+        
+        INPUT parameter is set to in_file.
+        OUTPUT parameter is set to a random file.
+        Any keyword args are passed as parameters.
+        """
+
+        pargs = []
+        for key, value in kwargs.items():
+            pargs.append('-p')
+            pargs.append('{}={}'.format(key, value))
+
+        out_file = tempfile.mktemp(dir=self._tmpdir)
+        params = [
+            '-f', script_path,
+            '-p', 'INPUT={}'.format(in_file),
+            '-p', 'OUTPUT={}'.format(out_file),
+            ] + pargs
+        self.run_pig_raw(params)
+        return out_file
+
diff --git a/pig/tests/test_filter_cdx.py b/pig/tests/test_filter_cdx.py
index 83f88bb..f46e5e1 100644
--- a/pig/tests/test_filter_cdx.py
+++ b/pig/tests/test_filter_cdx.py
@@ -10,9 +10,9 @@ Abstract into a base test class/template:
 import os
 import unittest
 from nose.tools import *
-from pigpy.hadoop import Hadoop
-
+from pighelper import PigTestHelper
 
+"""
 class TestFilterCDX(unittest.TestCase):
 
     def setUp(self):
@@ -33,3 +33,10 @@ class TestFilterCDX(unittest.TestCase):
         self.hadoop.run_pig_job("filter-cdx-ps.pig")
         self.hadoop.copyToLocal("/reports/output.csv", "output.csv")
 
+"""
+
+class TestFilterCDX(PigTestHelper):
+
+    def test_thing(self):
+
+        self.run_pig("filter-cdx-ps.pig", "tests/files/example.cdx")